Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

[pull] main from microsoft:main #683

Merged
merged 3 commits into from
Nov 27, 2024
Merged

[pull] main from microsoft:main #683

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
ef55044
Fix TryAddOptionSet (#2754)
LucGenetier Nov 27, 2024
b36ecc1
CDP - Remove SQL relationships (#2753)
LucGenetier Nov 27, 2024
216c29c
Disable flaky test (#2756)
MikeStall Nov 27, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
11 changes: 3 additions & 8 deletions src/libraries/Microsoft.PowerFx.Connectors/ConnectorFunction.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1010,7 +1010,7 @@ internal static ConnectorType GetConnectorType(string valuePath, StringValue sv,

// Only called by ConnectorTable.GetSchema
// Returns a FormulaType with AssociatedDataSources set (done in AddTabularDataSource)
internal static ConnectorType GetCdpTableType(ICdpTableResolver tableResolver, string connectorName, string tableName, string valuePath, StringValue stringValue, List<SqlRelationship> sqlRelationships, ConnectorCompatibility compatibility, string datasetName,
internal static ConnectorType GetCdpTableType(ICdpTableResolver tableResolver, string connectorName, string tableName, string valuePath, StringValue stringValue, ConnectorCompatibility compatibility, string datasetName,
out string name, out string displayName, out TableDelegationInfo delegationInfo, out IEnumerable<OptionSet> optionSets)
{
// There are some errors when parsing this Json payload but that's not a problem here as we only need x-ms-capabilities parsing to work
Expand All @@ -1025,7 +1025,7 @@ internal static ConnectorType GetCdpTableType(ICdpTableResolver tableResolver, s
IList<ReferencedEntity> referencedEntities = GetReferenceEntities(connectorName, stringValue);

SymbolTable symbolTable = new SymbolTable();
ConnectorType connectorType = new ConnectorType(jsonElement, tableName, symbolTable, compatibility, sqlRelationships, referencedEntities, datasetName, name, connectorName, tableResolver, serviceCapabilities, isTableReadOnly);
ConnectorType connectorType = new ConnectorType(jsonElement, tableName, symbolTable, compatibility, referencedEntities, datasetName, name, connectorName, tableResolver, serviceCapabilities, isTableReadOnly);
delegationInfo = ((DataSourceInfo)connectorType.FormulaType._type.AssociatedDataSources.First()).DelegationInfo;
optionSets = symbolTable.OptionSets.Select(kvp => kvp.Value);

Expand Down Expand Up @@ -1087,12 +1087,7 @@ private static ConnectorType GetConnectorTypeInternal(ConnectorCompatibility com
OpenApiSchema schema = new OpenApiStringReader(oars).ReadFragment<OpenApiSchema>(je.ToString(), OpenApi.OpenApiSpecVersion.OpenApi2_0, out OpenApiDiagnostic diag);

return new ConnectorType(SwaggerSchema.New(schema), compatibility);
}

private static ConnectorType GetJsonConnectorTypeInternal(ConnectorCompatibility compatibility, JsonElement je, IList<SqlRelationship> sqlRelationships)
{
return new ConnectorType(je, compatibility, sqlRelationships);
}
}

private async Task<ConnectorType> GetConnectorSuggestionsFromDynamicPropertyAsync(NamedValue[] knownParameters, BaseRuntimeConnectorContext runtimeContext, ConnectorDynamicProperty cdp, CancellationToken cancellationToken)
{
Expand Down
31 changes: 12 additions & 19 deletions src/libraries/Microsoft.PowerFx.Connectors/OpenApiExtensions.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -392,19 +392,17 @@ internal static void WhenPresent(this IDictionary<string, IOpenApiAny> apiObj, s

internal class ConnectorTypeGetterSettings
{
internal readonly ConnectorCompatibility Compatibility;
internal readonly IList<SqlRelationship> SqlRelationships;
internal readonly ConnectorCompatibility Compatibility;
internal Stack<string> Chain = new Stack<string>();
internal int Level = 0;
internal readonly SymbolTable OptionSets;

private readonly string _tableName;

internal ConnectorTypeGetterSettings(ConnectorCompatibility connectorCompatibility, string tableName, SymbolTable optionSets, IList<SqlRelationship> sqlRelationships = null)
internal ConnectorTypeGetterSettings(ConnectorCompatibility connectorCompatibility, string tableName, SymbolTable optionSets)
{
Compatibility = connectorCompatibility;
OptionSets = optionSets;
SqlRelationships = sqlRelationships;
OptionSets = optionSets;

_tableName = tableName;
}
Expand Down Expand Up @@ -437,14 +435,14 @@ internal string GetOptionSetName(string optionSetNameBase)
}
}

internal static ConnectorType GetConnectorType(this ISwaggerParameter openApiParameter, ConnectorCompatibility compatibility, IList<SqlRelationship> sqlRelationships = null)
internal static ConnectorType GetConnectorType(this ISwaggerParameter openApiParameter, ConnectorCompatibility compatibility)
{
return openApiParameter.GetConnectorType(tableName: null, optionSets: null, compatibility, sqlRelationships);
return openApiParameter.GetConnectorType(tableName: null, optionSets: null, compatibility);
}

internal static ConnectorType GetConnectorType(this ISwaggerParameter openApiParameter, string tableName, SymbolTable optionSets, ConnectorCompatibility compatibility, IList<SqlRelationship> sqlRelationships = null)
internal static ConnectorType GetConnectorType(this ISwaggerParameter openApiParameter, string tableName, SymbolTable optionSets, ConnectorCompatibility compatibility)
{
ConnectorTypeGetterSettings settings = new ConnectorTypeGetterSettings(compatibility, tableName, optionSets, sqlRelationships);
ConnectorTypeGetterSettings settings = new ConnectorTypeGetterSettings(compatibility, tableName, optionSets);
ConnectorType connectorType = openApiParameter.GetConnectorType(settings);

return connectorType;
Expand Down Expand Up @@ -681,16 +679,6 @@ internal static ConnectorType GetConnectorType(this ISwaggerParameter openApiPar
//ConnectorType propertyType = new OpenApiParameter() { Name = propLogicalName, Required = schema.Required.Contains(propLogicalName), Schema = kv.Value, Extensions = kv.Value.Extensions }.GetConnectorType(settings.Stack(schemaIdentifier));
ConnectorType propertyType = new SwaggerParameter(propLogicalName, schema.Required.Contains(propLogicalName), kv.Value, kv.Value.Extensions).GetConnectorType(settings.Stack(schemaIdentifier));

if (settings.SqlRelationships != null)
{
SqlRelationship relationship = settings.SqlRelationships.FirstOrDefault(sr => sr.ColumnName == propLogicalName);

if (relationship != null)
{
propertyType.SetRelationship(relationship);
}
}

settings.UnStack();

if (propertyType.HiddenRecordType != null)
Expand Down Expand Up @@ -734,6 +722,11 @@ internal static OptionSet TryAddOptionSet(this SymbolTable symbolTable, OptionSe
throw new ArgumentNullException("optionSet");
}

if (symbolTable == null)
{
return optionSet;
}

string name = optionSet.EntityName;

// No existing symbols with that name
Expand Down
19 changes: 3 additions & 16 deletions src/libraries/Microsoft.PowerFx.Connectors/Public/ConnectorType.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -204,16 +204,11 @@ internal ConnectorType(string error, ErrorResourceKey warning = default)
internal ConnectorType(ISwaggerSchema schema, ConnectorCompatibility compatibility)
: this(schema, null, new SwaggerParameter(null, true, schema, null).GetConnectorType(compatibility))
{
}

internal ConnectorType(JsonElement schema, ConnectorCompatibility compatibility, IList<SqlRelationship> sqlRelationships)
: this(SwaggerJsonSchema.New(schema), null, new SwaggerParameter(null, true, SwaggerJsonSchema.New(schema), null).GetConnectorType(compatibility, sqlRelationships))
{
}
}

// Called by ConnectorFunction.GetCdpTableType
internal ConnectorType(JsonElement schema, string tableName, SymbolTable optionSets, ConnectorCompatibility compatibility, IList<SqlRelationship> sqlRelationships, IList<ReferencedEntity> referencedEntities, string datasetName, string name, string connectorName, ICdpTableResolver resolver, ServiceCapabilities serviceCapabilities, bool isTableReadOnly)
: this(SwaggerJsonSchema.New(schema), null, new SwaggerParameter(null, true, SwaggerJsonSchema.New(schema), null).GetConnectorType(tableName, optionSets, compatibility, sqlRelationships))
internal ConnectorType(JsonElement schema, string tableName, SymbolTable optionSets, ConnectorCompatibility compatibility, IList<ReferencedEntity> referencedEntities, string datasetName, string name, string connectorName, ICdpTableResolver resolver, ServiceCapabilities serviceCapabilities, bool isTableReadOnly)
: this(SwaggerJsonSchema.New(schema), null, new SwaggerParameter(null, true, SwaggerJsonSchema.New(schema), null).GetConnectorType(tableName, optionSets, compatibility))
{
Name = name;

Expand Down Expand Up @@ -294,14 +289,6 @@ internal DisplayNameProvider DisplayNameProvider

private DisplayNameProvider _displayNameProvider;

internal void SetRelationship(SqlRelationship relationship)
{
ExternalTables ??= new List<string>();
ExternalTables.Add(relationship.ReferencedTable);
RelationshipName = relationship.RelationshipName;
ForeignKey = relationship.ReferencedColumnName;
}

private void AggregateErrors(ConnectorType[] types)
{
if (types != null)
Expand Down
133 changes: 70 additions & 63 deletions src/libraries/Microsoft.PowerFx.Connectors/Tabular/CdpTableResolver.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,6 @@
using System.Collections.Generic;
using System.Linq;
using System.Net.Http;
using System.Text.Json;
using System.Threading;
using System.Threading.Tasks;
using Microsoft.PowerFx.Core.Entities;
Expand Down Expand Up @@ -63,35 +62,39 @@ public async Task<ConnectorType> ResolveTableAsync(string tableName, Cancellatio
return null;
}

List<SqlRelationship> sqlRelationships = null;

// for SQL need to get relationships separately as they aren't included by CDP connector
if (IsSql(_uriPrefix))
{
cancellationToken.ThrowIfCancellationRequested();

uri = (_uriPrefix ?? string.Empty) + $"/v2/datasets/{dataset}/query/sql";
string body =
@"{""query"":""SELECT fk.name AS FK_Name, '[' + sp.name + '].[' + tp.name + ']' AS Parent_Table, cp.name AS Parent_Column, '[' + sr.name + '].[' + tr.name + ']' AS Referenced_Table, cr.name AS Referenced_Column" +
@" FROM sys.foreign_keys fk" +
@" INNER JOIN sys.tables tp ON fk.parent_object_id = tp.object_id" +
@" INNER JOIN sys.tables tr ON fk.referenced_object_id = tr.object_id" +
@" INNER JOIN sys.schemas sp on tp.schema_id = sp.schema_id" +
@" INNER JOIN sys.schemas sr on tr.schema_id = sr.schema_id" +
@" INNER JOIN sys.foreign_key_columns fkc ON fkc.constraint_object_id = fk.object_id" +
@" INNER JOIN sys.columns cp ON fkc.parent_column_id = cp.column_id AND fkc.parent_object_id = cp.object_id" +
@" INNER JOIN sys.columns cr ON fkc.referenced_column_id = cr.column_id AND fkc.referenced_object_id = cr.object_id" +
@" WHERE '[' + sp.name + '].[' + tp.name + ']' = '" + tableName + "'" + @"""}";

string text2 = await CdpServiceBase.GetObject(_httpClient, $"Get SQL relationships", uri, body, cancellationToken, Logger).ConfigureAwait(false);

// Result should be cached
sqlRelationships = GetSqlRelationships(text2);
}
// We don't need SQL relationships as those are not equivalent as those we find in Dataverse or ServiceNow
// Foreign Key constrainsts are not enough and equivalent.
// Only keeping code for future use, if we'd need to get those relationships.
//
//List<SqlRelationship> sqlRelationships = null;
//
//// for SQL need to get relationships separately as they aren't included by CDP connector
//if (IsSql(_uriPrefix))
//{
// cancellationToken.ThrowIfCancellationRequested();
//
// uri = (_uriPrefix ?? string.Empty) + $"/v2/datasets/{dataset}/query/sql";
// string body =
// @"{""query"":""SELECT fk.name AS FK_Name, '[' + sp.name + '].[' + tp.name + ']' AS Parent_Table, cp.name AS Parent_Column, '[' + sr.name + '].[' + tr.name + ']' AS Referenced_Table, cr.name AS Referenced_Column" +
// @" FROM sys.foreign_keys fk" +
// @" INNER JOIN sys.tables tp ON fk.parent_object_id = tp.object_id" +
// @" INNER JOIN sys.tables tr ON fk.referenced_object_id = tr.object_id" +
// @" INNER JOIN sys.schemas sp on tp.schema_id = sp.schema_id" +
// @" INNER JOIN sys.schemas sr on tr.schema_id = sr.schema_id" +
// @" INNER JOIN sys.foreign_key_columns fkc ON fkc.constraint_object_id = fk.object_id" +
// @" INNER JOIN sys.columns cp ON fkc.parent_column_id = cp.column_id AND fkc.parent_object_id = cp.object_id" +
// @" INNER JOIN sys.columns cr ON fkc.referenced_column_id = cr.column_id AND fkc.referenced_object_id = cr.object_id" +
// @" WHERE '[' + sp.name + '].[' + tp.name + ']' = '" + tableName + "'" + @"""}";
//
// string text2 = await CdpServiceBase.GetObject(_httpClient, $"Get SQL relationships", uri, body, cancellationToken, Logger).ConfigureAwait(false);
//
// // Result should be cached
// sqlRelationships = GetSqlRelationships(text2);
//}

string connectorName = _uriPrefix.Split(new[] { '/' }, StringSplitOptions.RemoveEmptyEntries)[1];

ConnectorType connectorType = ConnectorFunction.GetCdpTableType(this, connectorName, _tabularTable.TableName, "Schema/Items", FormulaValue.New(text), sqlRelationships, ConnectorCompatibility.CdpCompatibility, _tabularTable.DatasetName,
ConnectorType connectorType = ConnectorFunction.GetCdpTableType(this, connectorName, _tabularTable.TableName, "Schema/Items", FormulaValue.New(text), ConnectorCompatibility.CdpCompatibility, _tabularTable.DatasetName,
out string name, out string displayName, out TableDelegationInfo delegationInfo, out IEnumerable<OptionSet> optionSets);

OptionSets = optionSets;
Expand All @@ -104,47 +107,51 @@ public async Task<ConnectorType> ResolveTableAsync(string tableName, Cancellatio
internal static bool UseV2(string uriPrefix) => uriPrefix.Contains("/sql/") ||
uriPrefix.Contains("/zendesk/");

private List<SqlRelationship> GetSqlRelationships(string text)
{
RelationshipResult r = JsonSerializer.Deserialize<RelationshipResult>(text);

var relationships = r.ResultSets.Table1;
if (relationships == null || relationships.Length == 0)
{
return new List<SqlRelationship>();
}

List<SqlRelationship> sqlRelationShips = new List<SqlRelationship>();

foreach (var fk in relationships)
{
sqlRelationShips.Add(new SqlRelationship()
{
RelationshipName = fk.FK_Name,
ParentTable = fk.Parent_Table,
ColumnName = fk.Parent_Column,
ReferencedTable = fk.Referenced_Table,
ReferencedColumnName = fk.Referenced_Column
});
}

return sqlRelationShips;
}
// Only keeping code for reference
//
//private List<SqlRelationship> GetSqlRelationships(string text)
//{
// RelationshipResult r = JsonSerializer.Deserialize<RelationshipResult>(text);

// var relationships = r.ResultSets.Table1;
// if (relationships == null || relationships.Length == 0)
// {
// return new List<SqlRelationship>();
// }

// List<SqlRelationship> sqlRelationShips = new List<SqlRelationship>();

// foreach (var fk in relationships)
// {
// sqlRelationShips.Add(new SqlRelationship()
// {
// RelationshipName = fk.FK_Name,
// ParentTable = fk.Parent_Table,
// ColumnName = fk.Parent_Column,
// ReferencedTable = fk.Referenced_Table,
// ReferencedColumnName = fk.Referenced_Column
// });
// }

// return sqlRelationShips;
//}
}

#pragma warning disable SA1300 // Element should begin with upper case
#pragma warning disable SA1516 // Element should be separated by a blank line

internal class SqlRelationship
{
public string RelationshipName;
public string ParentTable;
public string ColumnName;
public string ReferencedTable;
public string ReferencedColumnName;

public override string ToString() => $"{RelationshipName}, {ParentTable}, {ColumnName}, {ReferencedTable}, {ReferencedColumnName}";
}
// Only keeping code for reference
//
//internal class SqlRelationship
//{
// public string RelationshipName;
// public string ParentTable;
// public string ColumnName;
// public string ReferencedTable;
// public string ReferencedColumnName;
//
// public override string ToString() => $"{RelationshipName}, {ParentTable}, {ColumnName}, {ReferencedTable}, {ReferencedColumnName}";
//}

internal class RelationshipResult
{
Expand Down
6 changes: 3 additions & 3 deletions src/tests/Microsoft.PowerFx.Connectors.Tests.Shared/CompatibilityTests.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -33,9 +33,9 @@ public void CompatibilityTest()

string text = (string)LoggingTestServer.GetFileText(@"Responses\Compatibility GetSchema.json");

ConnectorType ctCdp = ConnectorFunction.GetCdpTableType(tableResolver, "name", null, "schema/items", StringValue.New(text), null, ConnectorCompatibility.CdpCompatibility, "dataset", out _, out _, out _, out _);
ConnectorType ctPa = ConnectorFunction.GetCdpTableType(tableResolver, "name", null, "schema/items", StringValue.New(text), null, ConnectorCompatibility.PowerAppsCompatibility, "dataset", out _, out _, out _, out _);
ConnectorType ctSw = ConnectorFunction.GetCdpTableType(tableResolver, "name", null, "schema/items", StringValue.New(text), null, ConnectorCompatibility.SwaggerCompatibility, "dataset", out _, out _, out _, out _);
ConnectorType ctCdp = ConnectorFunction.GetCdpTableType(tableResolver, "name", null, "schema/items", StringValue.New(text), ConnectorCompatibility.CdpCompatibility, "dataset", out _, out _, out _, out _);
ConnectorType ctPa = ConnectorFunction.GetCdpTableType(tableResolver, "name", null, "schema/items", StringValue.New(text), ConnectorCompatibility.PowerAppsCompatibility, "dataset", out _, out _, out _, out _);
ConnectorType ctSw = ConnectorFunction.GetCdpTableType(tableResolver, "name", null, "schema/items", StringValue.New(text), ConnectorCompatibility.SwaggerCompatibility, "dataset", out _, out _, out _, out _);

string cdp = ctCdp.FormulaType.ToStringWithDisplayNames();
string pa = ctPa.FormulaType.ToStringWithDisplayNames();
Expand Down
Loading