From d10d68a5f9d232759547444e004c080a35b062d0 Mon Sep 17 00:00:00 2001 From: robgruen Date: Tue, 26 Nov 2024 20:49:12 -0800 Subject: [PATCH] Added nearby location lookup and reverse geocode lookup for images. (#436) Now when adding an image to the system we do a reverse geocode lookup we also perform a nearby POI search. --- ts/packages/aiclient/src/auth.ts | 1 + ts/packages/aiclient/src/openai.ts | 3 + ts/packages/commonUtils/package.json | 2 + ts/packages/commonUtils/src/image.ts | 4 +- ts/packages/commonUtils/src/indexNode.ts | 2 + ts/packages/commonUtils/src/jsonTranslator.ts | 38 +++- ts/packages/commonUtils/src/location.ts | 209 ++++++++++++++++++ ts/packages/commonUtils/test/location.spec.ts | 42 ++++ .../shell/src/main/shellSettingsType.ts | 2 +- .../shell/src/renderer/assets/styles.less | 6 +- ts/pnpm-lock.yaml | 80 +++++++ ts/tools/scripts/getKeys.config.json | 4 +- 12 files changed, 386 insertions(+), 7 deletions(-) create mode 100644 ts/packages/commonUtils/src/location.ts create mode 100644 ts/packages/commonUtils/test/location.spec.ts diff --git a/ts/packages/aiclient/src/auth.ts b/ts/packages/aiclient/src/auth.ts index d24cf261e..abde9c510 100644 --- a/ts/packages/aiclient/src/auth.ts +++ b/ts/packages/aiclient/src/auth.ts @@ -11,6 +11,7 @@ export interface AuthTokenProvider { export enum AzureTokenScopes { CogServices = "https://cognitiveservices.azure.com/.default", + AzureMaps = "https://atlas.microsoft.com/.default", } export function createAzureTokenProvider( diff --git a/ts/packages/aiclient/src/openai.ts b/ts/packages/aiclient/src/openai.ts index f50ecc6f1..6780a8580 100644 --- a/ts/packages/aiclient/src/openai.ts +++ b/ts/packages/aiclient/src/openai.ts @@ -95,6 +95,9 @@ export enum EnvVars { AZURE_OPENAI_ENDPOINT_DALLE = "AZURE_OPENAI_ENDPOINT_DALLE", OLLAMA_ENDPOINT = "OLLAMA_ENDPOINT", + + AZURE_MAPS_ENDPOINT = "AZURE_MAPS_ENDPOINT", + AZURE_MAPS_CLIENTID = "AZURE_MAPS_CLIENTID", } export const MAX_PROMPT_LENGTH_DEFAULT = 1000 * 60; diff --git a/ts/packages/commonUtils/package.json b/ts/packages/commonUtils/package.json index 26a39d087..71d66aced 100644 --- a/ts/packages/commonUtils/package.json +++ b/ts/packages/commonUtils/package.json @@ -27,6 +27,8 @@ "tsc": "tsc -b" }, "dependencies": { + "@azure-rest/maps-search": "^2.0.0-beta.2", + "@azure/maps-search": "^1.0.0-beta.2", "@typeagent/agent-sdk": "workspace:*", "aiclient": "workspace:*", "chalk": "^5.3.0", diff --git a/ts/packages/commonUtils/src/image.ts b/ts/packages/commonUtils/src/image.ts index 54f41732a..6a60de00a 100644 --- a/ts/packages/commonUtils/src/image.ts +++ b/ts/packages/commonUtils/src/image.ts @@ -25,7 +25,9 @@ export function extractRelevantExifTags(exifTags: ExifReader.Tags) { ${exifTags.DateTime ? "Date Taken: " + exifTags.DateTime.value : ""} ${exifTags.OffsetTime ? "Offset Time: " + exifTags.OffsetTime.value : ""} ${exifTags.GPSLatitude ? "GPS Latitude: " + exifTags.GPSLatitude.description : ""} - ${exifTags.GPSLongitude ? "GPS Longitude: " + exifTags.GPSLongitude.description : ""} + ${exifTags.GPSLatitudeRef ? "GPS Latitude Reference: " + exifTags.GPSLatitudeRef.value : ""} + ${exifTags.GPSLongitude ? "GPS Longitude Reference: " + exifTags.GPSLongitude.description : ""} + ${exifTags.GPSLongitudeRef ? "GPS Longitude Reference: " + exifTags.GPSLongitudeRef?.value : ""} ${exifTags.GPSAltitudeRef ? "GPS Altitude Reference: " + exifTags.GPSAltitudeRef.value : ""} ${exifTags.GPSAltitude ? "GPS Altitude: " + exifTags.GPSAltitude.description : ""} `; diff --git a/ts/packages/commonUtils/src/indexNode.ts b/ts/packages/commonUtils/src/indexNode.ts index d30c78f4d..58c728aec 100644 --- a/ts/packages/commonUtils/src/indexNode.ts +++ b/ts/packages/commonUtils/src/indexNode.ts @@ -39,3 +39,5 @@ export { } from "./mimeTypes.js"; export { getObjectProperty, setObjectProperty } from "./objectProperty.js"; + +export * from "./location.js"; diff --git a/ts/packages/commonUtils/src/jsonTranslator.ts b/ts/packages/commonUtils/src/jsonTranslator.ts index 06f181b72..ec2057738 100644 --- a/ts/packages/commonUtils/src/jsonTranslator.ts +++ b/ts/packages/commonUtils/src/jsonTranslator.ts @@ -22,6 +22,12 @@ import { IncrementalJsonValueCallBack, } from "./incrementalJsonParser.js"; import { CachedImageWithDetails, extractRelevantExifTags } from "./image.js"; +import { apiSettingsFromEnv } from "../../aiclient/dist/openai.js"; +import { + exifGPSTagToLatLong, + findNearbyPointsOfInterest, + reverseGeocode, +} from "./location.js"; export type InlineTranslatorSchemaDef = { kind: "inline"; @@ -152,7 +158,7 @@ export function enableJsonTranslatorStreaming( cb?: IncrementalJsonValueCallBack, attachments?: CachedImageWithDetails[], ) => { - attachAttachments(attachments, promptPreamble); + await attachAttachments(attachments, promptPreamble); return originalTranslate( request, initializeStreamingParser(promptPreamble, cb), @@ -162,7 +168,7 @@ export function enableJsonTranslatorStreaming( return translatorWithStreaming; } -function attachAttachments( +async function attachAttachments( attachments: CachedImageWithDetails[] | undefined, promptPreamble?: string | PromptSection[], ) { @@ -188,6 +194,34 @@ function attachAttachments( type: "text", text: `Image EXIF tags: \n${extractRelevantExifTags(attachments![i].exifTags)}`, }, + { + type: "text", + text: `Nearby Points of Interest: \n${JSON.stringify( + await findNearbyPointsOfInterest( + exifGPSTagToLatLong( + attachments![i].exifTags.GPSLatitude, + attachments![i].exifTags.GPSLatitudeRef, + attachments![i].exifTags.GPSLongitude, + attachments![i].exifTags.GPSLongitudeRef, + ), + apiSettingsFromEnv(), + ), + )}`, + }, + { + type: "text", + text: `Reverse Geocode Results: \n${JSON.stringify( + await reverseGeocode( + exifGPSTagToLatLong( + attachments![i].exifTags.GPSLatitude, + attachments![i].exifTags.GPSLatitudeRef, + attachments![i].exifTags.GPSLongitude, + attachments![i].exifTags.GPSLongitudeRef, + ), + apiSettingsFromEnv(), + ), + )}`, + }, ], }); } diff --git a/ts/packages/commonUtils/src/location.ts b/ts/packages/commonUtils/src/location.ts new file mode 100644 index 000000000..64ce51f65 --- /dev/null +++ b/ts/packages/commonUtils/src/location.ts @@ -0,0 +1,209 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +import { + AuthTokenProvider, + AzureTokenScopes, + createAzureTokenProvider, + getEnvSetting, +} from "aiclient"; +import { StringArrayTag, TypedTag, XmpTag } from "exifreader"; +import { ApiSettings, EnvVars } from "../../aiclient/dist/openai.js"; +import { env } from "process"; +import { + GeoJsonFeature, + GeoJsonFeatureCollection, + SearchAddressResult, + SearchAddressResultItem, +} from "@azure/maps-search"; +import { AddressOutput } from "@azure-rest/maps-search"; + +/** + * Point of interest + */ +export type PointOfInterest = { + name?: String | undefined; + categories?: String[] | undefined; + freeFormAddress?: String | undefined; + position?: LatLong | undefined; +}; + +/** + * Reverse geocode lookup + */ +export type ReverseGeocodeAddressLookup = { + address?: AddressOutput | undefined; + confidence?: "High" | "Medium" | "Low" | undefined; + type: any; +}; + +/** + * Latitude, longitude coordinates + */ +export type LatLong = { + latitude: Number | String | undefined; + longitude: Number | String | undefined; +}; + +/** + * Helper method to convert EXIF tags to Latlong type + * @param exifLat - The exif latitude. + * @param exifLatRef - The exif latitude reference. + * @param exifLong - The exif longitude + * @param exifLongRef - The exif longitude reference. + * @returns The LatLong represented by the EXIF tag or undefined when data is incomplete + */ +export function exifGPSTagToLatLong( + exifLat: + | XmpTag + | TypedTag<[[number, number], [number, number], [number, number]]> + | undefined, + exifLatRef: XmpTag | StringArrayTag | undefined, + exifLong: + | XmpTag + | TypedTag<[[number, number], [number, number], [number, number]]> + | undefined, + exifLongRef: XmpTag | StringArrayTag | undefined, +): LatLong | undefined { + if ( + exifLat !== undefined && + exifLong !== undefined && + exifLatRef !== undefined && + exifLongRef !== undefined + ) { + return { + latitude: + exifLatRef.value == "S" + ? parseFloat("-" + exifLat.description) + : parseFloat(exifLat.description), + longitude: + exifLongRef.value == "W" + ? parseFloat("-" + exifLong.description) + : parseFloat(exifLong.description), + }; + } + + return undefined; +} + +/** + * Gets the nearby POIs for the supplied coordinate and search radius + * @param position - the position at which to do a nearby search + * @param settings - the API settings containing the endpoint to call + * @param radius - the search radius + * @returns A list of summarized nearby POIs + */ +export async function findNearbyPointsOfInterest( + position: LatLong | undefined, + settings: ApiSettings, + radius: Number = 10, +): Promise { + if (position === undefined) { + return []; + } + + const tokenProvider: AuthTokenProvider = createAzureTokenProvider( + AzureTokenScopes.AzureMaps, + ); + const tokenResult = await tokenProvider.getAccessToken(); + if (!tokenResult.success) { + return []; + } + + try { + //let fuzzySearch = `${getEnvSetting(env, EnvVars.AZURE_MAPS_ENDPOINT)}search/fuzzy/json?api-version=1.0&query={lat,long}` + //let poi = `${getEnvSetting(env, EnvVars.AZURE_MAPS_ENDPOINT)}search/poi/{format}?api-version=1.0&lat={LAT}&lon={LON}` + const nearby = `${getEnvSetting(env, EnvVars.AZURE_MAPS_ENDPOINT)}search/nearby/json?api-version=1.0&lat=${position.latitude}&lon=${position.longitude}&radius=${radius}`; + const options: RequestInit = { + method: "GET", + headers: new Headers({ + Authorization: `Bearer ${tokenResult.data}`, + "x-ms-client-id": `${getEnvSetting(env, EnvVars.AZURE_MAPS_CLIENTID)}`, + }), + }; + + // get the result + const response = await fetch(nearby, options); + let responseBody = await response.json(); + + // summarize results + const results: SearchAddressResult = + responseBody as SearchAddressResult; + const retVal: PointOfInterest[] = []; + results.results.map((result: SearchAddressResultItem) => { + if (result.type == "POI") { + retVal.push({ + name: result.pointOfInterest?.name, + categories: result.pointOfInterest?.categories, + freeFormAddress: result.address.freeformAddress, + position: { + latitude: result.position[0], + longitude: result.position[1], + }, + }); + } else { + // TODO: handle more result types + throw new Error("Unknown nearby search result!"); + } + }); + + // TODO: if there are no POI, can we just send back the address? + // Do we increase POI search radius until we find something in some predefined maximum area? + return retVal; + } catch (e) { + console.warn(`Error performing nearby POI lookup: ${e}`); + return []; + } +} + +export async function reverseGeocode( + position: LatLong | undefined, + settings: ApiSettings, +): Promise { + if (position === undefined) { + return []; + } + + const tokenProvider: AuthTokenProvider = createAzureTokenProvider( + AzureTokenScopes.AzureMaps, + ); + const tokenResult = await tokenProvider.getAccessToken(); + if (!tokenResult.success) { + return []; + } + + try { + let reverseGeocode = `${getEnvSetting(env, EnvVars.AZURE_MAPS_ENDPOINT)}reverseGeocode?api-version=2023-06-01&coordinates=${position.longitude},${position.latitude}`; + + const options: RequestInit = { + method: "GET", + headers: new Headers({ + Authorization: `Bearer ${tokenResult.data}`, + "x-ms-client-id": `${getEnvSetting(env, EnvVars.AZURE_MAPS_CLIENTID)}`, + }), + }; + + // get the result + const response = await fetch(reverseGeocode, options); + let responseBody = await response.json(); + + // summarize results + const results: GeoJsonFeatureCollection = + responseBody as GeoJsonFeatureCollection; + const retVal: ReverseGeocodeAddressLookup[] = []; + results.features?.map((result: GeoJsonFeature) => { + if (result.properties !== undefined) { + retVal.push({ + address: result.properties.address, + confidence: result.properties.confidence, + type: result.properties.type, + }); + } + }); + + return retVal; + } catch (e) { + console.warn(`Unable to perform reverse geocode lookup: ${e}`); + return []; + } +} diff --git a/ts/packages/commonUtils/test/location.spec.ts b/ts/packages/commonUtils/test/location.spec.ts new file mode 100644 index 000000000..558c5d193 --- /dev/null +++ b/ts/packages/commonUtils/test/location.spec.ts @@ -0,0 +1,42 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +import { XmpTag } from "exifreader"; +import { exifGPSTagToLatLong, LatLong } from "../src/location.js"; + +describe("Location Tests", () => { + it("EXIF LatLong to LatLong", () => { + const lat: XmpTag = { + value: "47.6204", + description: "47.6204", + attributes: {}, + }; + const long: XmpTag = { + value: "122.3491", + description: "122.3491", + attributes: {}, + }; + const latRef: XmpTag = { + value: "N", + description: "North Latitude", + attributes: {}, + }; + const longRef: XmpTag = { + value: "W", + description: "West Longitude", + attributes: {}, + }; + + const ll: LatLong = exifGPSTagToLatLong(lat, latRef, long, longRef)!; + expect(ll.latitude == "47.6204"); + expect(ll.longitude == "-122.3491"); + + const llu: LatLong | undefined = exifGPSTagToLatLong( + lat, + latRef, + undefined, + longRef, + ); + expect(llu === undefined); + }); +}); diff --git a/ts/packages/shell/src/main/shellSettingsType.ts b/ts/packages/shell/src/main/shellSettingsType.ts index 415631abd..726ed6849 100644 --- a/ts/packages/shell/src/main/shellSettingsType.ts +++ b/ts/packages/shell/src/main/shellSettingsType.ts @@ -31,7 +31,7 @@ export const defaultSettings: ShellSettingsType = { notifyFilter: "error;warning;", tts: false, ttsSettings: {}, - agentGreeting: false, + agentGreeting: true, multiModalContent: true, devUI: false, partialCompletion: true, diff --git a/ts/packages/shell/src/renderer/assets/styles.less b/ts/packages/shell/src/renderer/assets/styles.less index 3f33f40c0..ffe35c586 100644 --- a/ts/packages/shell/src/renderer/assets/styles.less +++ b/ts/packages/shell/src/renderer/assets/styles.less @@ -165,8 +165,10 @@ body { padding: 5px; background-color: @chat-input-bg; align-self: flex-end; - margin: 10px 10px 10px 20px; - width: calc(100% - 40px); + margin: 10px 10px 10px 10px; + width: calc( + 100% - 20px + ); // Note: subtracted value should match horizontal margin total from line above border: solid 1px rgb(224, 224, 224); border-radius: 6px; position: relative; diff --git a/ts/pnpm-lock.yaml b/ts/pnpm-lock.yaml index 525564272..94ac5a77d 100644 --- a/ts/pnpm-lock.yaml +++ b/ts/pnpm-lock.yaml @@ -1411,6 +1411,12 @@ importers: packages/commonUtils: dependencies: + '@azure-rest/maps-search': + specifier: ^2.0.0-beta.2 + version: 2.0.0-beta.2 + '@azure/maps-search': + specifier: ^1.0.0-beta.2 + version: 1.0.0-beta.2 '@typeagent/agent-sdk': specifier: workspace:* version: link:../agentSdk @@ -1939,6 +1945,14 @@ packages: resolution: {integrity: sha512-lFMjJTrFL3j7L9yBxwYfCq2k6qqwHyzuUl/XBnif78PWTJYyL/dfowQHWE3sp6U6ZzqWiiIZnpTMO96zhkjwtg==} engines: {node: '>=6.0.0'} + '@azure-rest/core-client@1.4.0': + resolution: {integrity: sha512-ozTDPBVUDR5eOnMIwhggbnVmOrka4fXCs8n8mvUo4WLLc38kki6bAOByDoVZZPz/pZy2jMt2kwfpvy/UjALj6w==} + engines: {node: '>=18.0.0'} + + '@azure-rest/maps-search@2.0.0-beta.2': + resolution: {integrity: sha512-OS3WUxmIPCgccZdSwiZpFl6UbmyvbEuVVWb9F4GC+MglJ3WdOpBEf2CJ+QeKJw9LLCwpsf+8ySGNhEnzAHj1gQ==} + engines: {node: '>=18.0.0'} + '@azure/abort-controller@1.1.0': resolution: {integrity: sha512-TrRLIoSQVzfAJX9H1JeFjzAoDGcoK1IYX1UImfceTZpsyYfWr09Ss1aHW1y5TrrR3iq6RZLBwJ3E24uwPhwahw==} engines: {node: '>=12.0.0'} @@ -1959,6 +1973,10 @@ packages: resolution: {integrity: sha512-hHYFx9lz0ZpbO5W+iotU9tmIX1jPcoIjYUEUaWGuMi1628LCQ/z05TUR4P+NRtMgyoHQuyVYyGQiD3PC47kaIA==} engines: {node: '>=18.0.0'} + '@azure/core-lro@2.7.2': + resolution: {integrity: sha512-0YIpccoX8m/k00O7mDDMdJpbr6mf1yWo2dfmxt5A8XVZVVMz2SSKaEbMCeJRvgQ0IaSlqhjT47p4hVIRRy90xw==} + engines: {node: '>=18.0.0'} + '@azure/core-paging@1.6.2': resolution: {integrity: sha512-YKWi9YuCU04B55h25cnOYZHxXYtEvQEbKST5vqRga7hWY9ydd3FZHdeQF8pyh+acWZvppw13M/LMGx0LABUVMA==} engines: {node: '>=18.0.0'} @@ -1987,6 +2005,15 @@ packages: resolution: {integrity: sha512-/+4TtokaGgC+MnThdf6HyIH9Wrjp+CnCn3Nx3ggevN7FFjjNyjqg0yLlc2i9S+Z2uAzI8GYOo35Nzb1MhQ89MA==} engines: {node: '>=18.0.0'} + '@azure/maps-common@1.0.0-beta.2': + resolution: {integrity: sha512-PB9GlnfojcQ4nf9WXdQvWeAk7gm8P74o+Z5IHz5YLK/W+3vrNrmVVVuFpGOvCPrLjag50UinaZsMBtPtxoiobg==} + engines: {node: '>=14.0.0'} + + '@azure/maps-search@1.0.0-beta.2': + resolution: {integrity: sha512-l2tFiQP+jsRpJG6YtoFBuaiH1aYE9gewzAUQVzUf8KqqG+H7arUkfa7E2zA96nBg1b59TdPaDvWtcxL2ytiD3Q==} + engines: {node: '>=14.0.0'} + deprecated: Please use @azure-rest/maps-search + '@azure/msal-browser@3.13.0': resolution: {integrity: sha512-fD906nmJei3yE7la6DZTdUtXKvpwzJURkfsiz9747Icv4pit77cegSm6prJTKLQ1fw4iiZzrrWwxnhMLrTf5gQ==} engines: {node: '>=0.8.0'} @@ -7921,6 +7948,29 @@ snapshots: '@jridgewell/gen-mapping': 0.3.5 '@jridgewell/trace-mapping': 0.3.25 + '@azure-rest/core-client@1.4.0': + dependencies: + '@azure/abort-controller': 2.1.1 + '@azure/core-auth': 1.7.1 + '@azure/core-rest-pipeline': 1.15.1 + '@azure/core-tracing': 1.1.1 + '@azure/core-util': 1.8.1 + tslib: 2.6.2 + transitivePeerDependencies: + - supports-color + + '@azure-rest/maps-search@2.0.0-beta.2': + dependencies: + '@azure-rest/core-client': 1.4.0 + '@azure/core-auth': 1.7.1 + '@azure/core-lro': 2.7.2 + '@azure/core-rest-pipeline': 1.15.1 + '@azure/logger': 1.1.1 + '@azure/maps-common': 1.0.0-beta.2 + tslib: 2.6.2 + transitivePeerDependencies: + - supports-color + '@azure/abort-controller@1.1.0': dependencies: tslib: 2.6.2 @@ -7957,6 +8007,13 @@ snapshots: transitivePeerDependencies: - supports-color + '@azure/core-lro@2.7.2': + dependencies: + '@azure/abort-controller': 2.1.1 + '@azure/core-util': 1.8.1 + '@azure/logger': 1.1.1 + tslib: 2.6.2 + '@azure/core-paging@1.6.2': dependencies: tslib: 2.6.2 @@ -8017,6 +8074,29 @@ snapshots: dependencies: tslib: 2.6.2 + '@azure/maps-common@1.0.0-beta.2': + dependencies: + '@azure/abort-controller': 1.1.0 + '@azure/core-auth': 1.7.1 + '@azure/core-client': 1.9.1 + '@azure/core-lro': 2.7.2 + '@azure/core-rest-pipeline': 1.15.1 + transitivePeerDependencies: + - supports-color + + '@azure/maps-search@1.0.0-beta.2': + dependencies: + '@azure/core-auth': 1.7.1 + '@azure/core-client': 1.9.1 + '@azure/core-lro': 2.7.2 + '@azure/core-rest-pipeline': 1.15.1 + '@azure/core-tracing': 1.1.1 + '@azure/logger': 1.1.1 + '@azure/maps-common': 1.0.0-beta.2 + tslib: 2.6.2 + transitivePeerDependencies: + - supports-color + '@azure/msal-browser@3.13.0': dependencies: '@azure/msal-common': 14.9.0 diff --git a/ts/tools/scripts/getKeys.config.json b/ts/tools/scripts/getKeys.config.json index 4a7cdf422..ff8156d59 100644 --- a/ts/tools/scripts/getKeys.config.json +++ b/ts/tools/scripts/getKeys.config.json @@ -31,7 +31,9 @@ "MSGRAPH_APP_CLIENTSECRET", "MSGRAPH_APP_TENANTID", "BING_API_KEY", - "BING_MAPS_API_KEY" + "BING_MAPS_API_KEY", + "AZURE_MAPS_ENDPOINT", + "AZURE_MAPS_CLIENTID" ], "private": ["SPOTIFY_APP_CLI", "SPOTIFY_APP_CLISEC", "SPOTIFY_APP_PORT"], "delete": ["MSGRAPH_APP_PASSWD", "MSGRAPH_APP_USERNAME"]