You should be able to use System.Net.WebClient to send a request to a server with a url compliant with Esri's REST API.
For example, to retrieve a the feature whose ObjectID=2, passing this url:
http://sampleserver1.arcgisonline.com/ArcGIS/rest/services/Specialty/ESRI_StatesCitiesRivers_USA/MapServer/0/2?f=pjson
The server returns a response that includes attributes and geometry:
{
"feature" :
{
"attributes" : {
"FID" : 2,
"CITY_FIPS" : "01990",
"CITY_NAME" : "Anacortes",
"STATE_FIPS" : "53",
"STATE_NAME" : "Washington",
"STATE_CITY" : "5301990",
"TYPE" : "city",
"CAPITAL" : "N",
"ELEVATION" : -99,
"POP1990" : 11451,
"HOUSEHOLDS" : 4669,
"MALES" : 5506,
"FEMALES" : 5945,
"WHITE" : 10945,
"BLACK" : 62,
"AMERI_ES" : 192,
"ASIAN_PI" : 154,
"OTHER" : 98,
"HISPANIC" : 233,
"AGE_UNDER5" : 725,
"AGE_5_17" : 1981,
"AGE_18_64" : 6276,
"AGE_65_UP" : 2469,
"NEVERMARRY" : 1420,
"MARRIED" : 5818,
"SEPARATED" : 151,
"WIDOWED" : 833,
"DIVORCED" : 916,
"HSEHLD_1_M" : 437,
"HSEHLD_1_F" : 711,
"MARHH_CHD" : 1032,
"MARHH_NO_C" : 1777,
"MHH_CHILD" : 76,
"FHH_CHILD" : 255,
"HSE_UNITS" : 4992,
"VACANT" : 323,
"OWNER_OCC" : 3181,
"RENTER_OCC" : 1488,
"MEDIAN_VAL" : 85300,
"MEDIANRENT" : 342,
"UNITS_1DET" : 3724,
"UNITS_1ATT" : 121,
"UNITS2" : 134,
"UNITS3_9" : 380,
"UNITS10_49" : 353,
"UNITS50_UP" : 0,
"MOBILEHOME" : 200
}
,
"geometry" :
{
"x" : -122.630685404,
"y" : 48.492215849
}
}
}
ArcObjects cannot be used within a silverlight app. If you want to turn a resulting REST geometry into an ArcObjects IGeometry, I would use Esri's WPF library to parse the REST, then convert the WPF geometry into an IGeometry.