Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Added QNN EP for ARM64 builds. #41

Merged
merged 15 commits into from
Dec 13, 2024
Merged

Added QNN EP for ARM64 builds. #41

Show file tree
Hide file tree
Changes from 5 commits
Commits
Show all changes
15 commits
Select commit Hold shift + click to select a range
144afab
Added QNN EP for ARM64 builds.
azchohfi Nov 26, 2024
0694dc1
Added QNN EP.
azchohfi Nov 26, 2024
df1dc6d
Added QNN for pose model and embeddings.
azchohfi Nov 26, 2024
ed6ab9b
Merge branch 'main' into alzollin/qnn
azchohfi Nov 27, 2024
d88efc2
Merge branch 'main' into alzollin/qnn
nmetulev Nov 27, 2024
05916db
Merge branch 'main' into alzollin/qnn
azchohfi Dec 6, 2024
182ace8
Merge branch 'main' into alzollin/qnn
niels9001 Dec 9, 2024
8c918af
Merge branch 'main' into alzollin/qnn
azchohfi Dec 12, 2024
0e6c68c
Disabled DML on QC, removed QNN from embeddings, minor cleanup
nmetulev Dec 13, 2024
cf53a56
Added hardware accelerator as a param to sample generator.
azchohfi Dec 13, 2024
cf1fc07
Merge branch 'main' into alzollin/qnn
azchohfi Dec 13, 2024
ddfd3c7
Fixed generator for ARM64/QNN.
azchohfi Dec 13, 2024
a5f6945
Merge branch 'dev/npu' into alzollin/qnn
nmetulev Dec 13, 2024
a566665
Merge branch 'dev/npu' into alzollin/qnn
nmetulev Dec 13, 2024
d6cdd72
Merge branch 'dev/npu' into alzollin/qnn
nmetulev Dec 13, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 2 additions & 1 deletion AIDevGallery.SourceGenerator/Models/HardwareAccelerator.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,7 @@ namespace AIDevGallery.SourceGenerator.Models
internal enum HardwareAccelerator
{
CPU,
DML
DML,
QNN
}
}
15 changes: 13 additions & 2 deletions AIDevGallery/AIDevGallery.csproj
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -57,7 +57,6 @@
<PackageReference Include="CommunityToolkit.WinUI.Media" />
<PackageReference Include="CommunityToolkit.WinUI.UI.Controls.Markdown" />
<PackageReference Include="MathNet.Numerics" />
<PackageReference Include="Microsoft.AI.DirectML" />
<PackageReference Include="Microsoft.Build" />
<PackageReference Include="Microsoft.Extensions.AI.Abstractions" />
<PackageReference Include="Microsoft.ML.OnnxRuntime.Extensions" />
Expand All @@ -83,7 +82,19 @@
<Manifest Include="$(ApplicationManifest)" />
</ItemGroup>

<ItemGroup>
<ItemGroup Condition="$(Platform) == 'ARM64'">
<PackageReference Include="Microsoft.ML.OnnxRuntime.Qnn" />
<PackageReference Include="Microsoft.ML.OnnxRuntimeGenAI" GeneratePathProperty="true" ExcludeAssets="all" />
<None Include="$(PKGMicrosoft_ML_OnnxRuntimeGenAI)\runtimes\win-arm64\native\onnxruntime-genai.dll">
<Link>onnxruntime-genai.dll</Link>
<CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
<Visible>false</Visible>
</None>
<PackageReference Include="Microsoft.ML.OnnxRuntimeGenAI.Managed" />
</ItemGroup>

<ItemGroup Condition="$(Platform) == 'x64'">
<PackageReference Include="Microsoft.AI.DirectML" />
<PackageReference Include="Microsoft.ML.OnnxRuntime.DirectML" />
<PackageReference Include="Microsoft.ML.OnnxRuntimeGenAI.DirectML" />
</ItemGroup>
Expand Down
3 changes: 2 additions & 1 deletion AIDevGallery/Models/ModelCompatibility.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -23,7 +23,8 @@ public static ModelCompatibility GetModelCompatibility(ModelDetails modelDetails

ModelCompatibilityState compatibility;
if (modelDetails.HardwareAccelerators.Contains(HardwareAccelerator.CPU) ||
(modelDetails.HardwareAccelerators.Contains(HardwareAccelerator.DML) && modelDetails.SupportedOnQualcomm == true))
(modelDetails.HardwareAccelerators.Contains(HardwareAccelerator.DML) && modelDetails.SupportedOnQualcomm == true) ||
nmetulev marked this conversation as resolved.
Show resolved Hide resolved
(modelDetails.HardwareAccelerators.Contains(HardwareAccelerator.QNN) && DeviceUtils.IsArm64()))
{
compatibility = ModelCompatibilityState.Compatible;
}
Expand Down
3 changes: 2 additions & 1 deletion AIDevGallery/Models/Samples.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -152,7 +152,8 @@ internal class Scenario
internal enum HardwareAccelerator
{
CPU,
DML
DML,
QNN
}
}

Expand Down
3 changes: 2 additions & 1 deletion AIDevGallery/ProjectGenerator/Template/HardwareAccelerator.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,5 +3,6 @@ namespace $safeprojectname$.SharedCode;
internal enum HardwareAccelerator
{
CPU,
DML
DML,
QNN
}
6 changes: 4 additions & 2 deletions AIDevGallery/Samples/ModelsDefinitions/embeddings.modelgroup.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,8 @@
"Description": "This is an embedding model executed on the CPU or GPU",
"HardwareAccelerator": [
"CPU",
"DML"
"DML",
"QNN"
nmetulev marked this conversation as resolved.
Show resolved Hide resolved
],
"SupportedOnQualcomm": true,
"Size": 90636722,
Expand All @@ -44,7 +45,8 @@
"Description": "This is an embedding model executed on the CPU or GPU",
"HardwareAccelerator": [
"CPU",
"DML"
"DML",
"QNN"
],
"SupportedOnQualcomm": true,
"Size": 133358075,
Expand Down
3 changes: 2 additions & 1 deletion AIDevGallery/Samples/ModelsDefinitions/imagemodels.modelgroup.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -132,7 +132,8 @@
"Description": "HRNet Pose",
"HardwareAccelerator": [
"CPU",
"DML"
"DML",
"QNN"
],
"SupportedOnQualcomm": true,
"Size": 114232171,
Expand Down
8 changes: 8 additions & 0 deletions AIDevGallery/Samples/Open Source Models/Image Models/HRNetPose/PoseDetection.xaml.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -76,6 +76,14 @@ private Task InitModel(string modelPath, HardwareAccelerator hardwareAccelerator
{
sessionOptions.AppendExecutionProvider_DML(DeviceUtils.GetBestDeviceId());
}
else if (hardwareAccelerator == HardwareAccelerator.QNN)
{
Dictionary<string, string> options = new()
{
{ "backend_path", "QnnHtp.dll" }
};
sessionOptions.AppendExecutionProvider("QNN", options);
}

_inferenceSession = new InferenceSession(modelPath, sessionOptions);
});
Expand Down
14 changes: 11 additions & 3 deletions AIDevGallery/Samples/SharedCode/EmbeddingGenerator.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -32,6 +32,7 @@ internal partial class EmbeddingGenerator : IDisposable, IEmbeddingGenerator<str
private readonly SessionOptions _sessionOptions;
private readonly InferenceSession _inferenceSession;
private readonly BertTokenizer _tokenizer;
private readonly int _chunkSize = 128;

public EmbeddingGeneratorMetadata Metadata { get; }
public EmbeddingGenerator(string modelPath, HardwareAccelerator hardwareAccelerator)
Expand All @@ -44,6 +45,15 @@ public EmbeddingGenerator(string modelPath, HardwareAccelerator hardwareAccelera
{
_sessionOptions.AppendExecutionProvider_DML(DeviceUtils.GetBestDeviceId());
}
else if (hardwareAccelerator == HardwareAccelerator.QNN)
{
Dictionary<string, string> options = new()
{
{ "backend_path", "QnnHtp.dll" }
nmetulev marked this conversation as resolved.
Show resolved Hide resolved
};
_sessionOptions.AppendExecutionProvider("QNN", options);
_chunkSize = 8;
}

_inferenceSession = new InferenceSession(Path.Join(modelPath, "onnx", "model.onnx"), _sessionOptions);
_tokenizer = BertTokenizer.Create(Path.Join(modelPath, "vocab.txt"));
Expand Down Expand Up @@ -103,9 +113,7 @@ public async IAsyncEnumerable<Embedding<float>> GenerateStreamingAsync(
EmbeddingGenerationOptions? options = null,
[EnumeratorCancellation] CancellationToken cancellationToken = default)
{
int chunkSize = 128;

var chunks = values.Chunk(chunkSize);
var chunks = values.Chunk(_chunkSize);

using var runOptions = new RunOptions();

Expand Down
5 changes: 2 additions & 3 deletions AIDevGallery/Utils/AppUtils.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -78,11 +78,10 @@ public static string GetHardwareAcceleratorString(HardwareAccelerator hardwareAc
{
switch (hardwareAccelerator)
{
default:
case HardwareAccelerator.CPU:
return "CPU";
case HardwareAccelerator.DML:
return "GPU";
default:
return hardwareAccelerator.ToString();
}
}

Expand Down
5 changes: 4 additions & 1 deletion Directory.Packages.props
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -18,8 +18,11 @@
<PackageVersion Include="CommunityToolkit.WinUI.Converters" Version="8.1.240916" />
<PackageVersion Include="CommunityToolkit.WinUI.UI.Controls.Markdown" Version="7.1.2" />
<PackageVersion Include="Microsoft.Build" Version="17.11.4" />
<PackageVersion Include="Microsoft.ML.OnnxRuntime.DirectML" Version="1.20.0" />
<PackageVersion Include="Microsoft.ML.OnnxRuntime.DirectML" Version="1.20.1" />
<PackageVersion Include="Microsoft.ML.OnnxRuntime.Qnn" Version="1.20.1" />
<PackageVersion Include="Microsoft.ML.OnnxRuntime.Extensions" Version="0.13.0" />
<PackageVersion Include="Microsoft.ML.OnnxRuntimeGenAI" Version="0.5.2" />
<PackageVersion Include="Microsoft.ML.OnnxRuntimeGenAI.Managed" Version="0.5.2" />
<PackageVersion Include="Microsoft.ML.OnnxRuntimeGenAI.DirectML" Version="0.5.0" />
<PackageVersion Include="Microsoft.WindowsAppSDK" Version="1.6.241106002" />
<PackageVersion Include="CommunityToolkit.WinUI.Animations" Version="8.1.240916" />
Expand Down