Merge branch 'master' of https://github.com/QutEcoacoustics/audio-ana…

…lysis

AudioAnalysis.sln

@@ -332,8 +332,8 @@ Global
 		{7C6831FD-F60C-4F7B-8E6A-35E850174411}.Debug|x86.Build.0 = Debug|x86
 		{7C6831FD-F60C-4F7B-8E6A-35E850174411}.Release|Any CPU.ActiveCfg = Release|Any CPU
 		{7C6831FD-F60C-4F7B-8E6A-35E850174411}.Release|Any CPU.Build.0 = Release|Any CPU
-		{7C6831FD-F60C-4F7B-8E6A-35E850174411}.Release|Mixed Platforms.ActiveCfg = Release|x86
-		{7C6831FD-F60C-4F7B-8E6A-35E850174411}.Release|Mixed Platforms.Build.0 = Release|x86
+		{7C6831FD-F60C-4F7B-8E6A-35E850174411}.Release|Mixed Platforms.ActiveCfg = Release|Any CPU
+		{7C6831FD-F60C-4F7B-8E6A-35E850174411}.Release|Mixed Platforms.Build.0 = Release|Any CPU
 		{7C6831FD-F60C-4F7B-8E6A-35E850174411}.Release|x64.ActiveCfg = Release|Any CPU
 		{7C6831FD-F60C-4F7B-8E6A-35E850174411}.Release|x64.Build.0 = Release|Any CPU
 		{7C6831FD-F60C-4F7B-8E6A-35E850174411}.Release|x86.ActiveCfg = Release|x86

appveyor.yml

@@ -37,25 +37,35 @@ environment:
   # An encrypted token that allows deploying to GitHub (registered to @atruskie)
   GH_CREATE_RELEASES_TOKEN:
     secure: iVfuU5KWiUqXeahc+j7hfQPquJFuxRXnLJajRBE94PGxkliT2GGaXkxJGJVCBDPW
+  # These are the personal credentials of someone with access to the personal
+  # account / organization that we want to push to
+  DOCKER_USER:
+    secure: A6mk2EeHQsCrlZnK7vwzAA==
+  DOCKER_PASS:
+    secure: eTD4d8vJCQW3CCNqGLKU2e79BhEeBVt6TxFMjy3f/KQ=
 
 # Before the clone occurs
 init:
   #- ps: iex ((new-object net.webclient).DownloadString('https://raw.githubusercontent.com/appveyor/ci/master/scripts/enable-rdp.ps1'))
   - echo Starting Build
-  #- git lfs install --skip-smudge
 
 before_build:
   # Download fresh lfs assets
   - git lfs pull
   # Determine whether or not we should tag and release
   # if the last tag hash matches the current build hash then we should not duplicate a release
-  - ps: $env:TAG_NOT_PREVIOUSLY_RELEASED = ($(git tag -l --format="%(objectname)") -notcontains $env:APPVEYOR_REPO_COMMIT)
-  - ps: $env:RELEASE_COMMIT = $env:APPVEYOR_REPO_COMMIT_MESSAGE_EXTENDED -ilike "*``[release``]*"; $env:SHOULD_RELEASE = ($env:RELEASE_COMMIT -ieq "true") -or ($env:APPVEYOR_SCHEDULED_BUILD -and ($env:TAG_NOT_PREVIOUSLY_RELEASED -ieq "true"));
-  - ps: ls env:* -Include SHOULD*,APPVEYOR_SCHEDULED_BUILD*,APPVEYOR_REPO_COMMIT_MESSAGE_EXTENDED*,RELEASE_COMMIT,TAG_NOT_PREVIOUSLY_RELEASED
+  - ps: |
+      $tags = git log --tags -n 10 --no-walk --format="%H %D"
+      Write-Output ("Recent tags:`n" + ($tags | Out-String))
+      $env:TAG_NOT_PREVIOUSLY_RELEASED=($tags -like "$env:APPVEYOR_REPO_COMMIT*").Length -eq 0
+      $env:RELEASE_COMMIT = $env:APPVEYOR_REPO_COMMIT_MESSAGE_EXTENDED -ilike "*``[release``]*"
+      $env:SHOULD_RELEASE = ($env:RELEASE_COMMIT -ieq "true") -or ($env:APPVEYOR_SCHEDULED_BUILD -and ($env:TAG_NOT_PREVIOUSLY_RELEASED -ieq "true"))
+      ls env:* -Include SHOULD*,APPVEYOR_SCHEDULED_BUILD*,APPVEYOR_REPO_COMMIT_MESSAGE_EXTENDED*,RELEASE_COMMIT,TAG_NOT_PREVIOUSLY_RELEASED
   # Restore packages 
   - nuget restore
   # Debugging a dirty flag in build version
   - git status
+  - docker version
 
 build_script:
   - echo "Building Debug"
@@ -118,13 +128,19 @@ deploy:
     description: "$(ApReleaseMessage)"
     auth_token: $(GH_CREATE_RELEASES_TOKEN)
     artifact: DebugPackage,ReleasePackage
-    prerelease: true
+    prerelease: false
     on:
       # Do not create new releases unless this is the master branch
       branch: master
       # Do not create a new release unless this was a scheduled build or a forced build
       SHOULD_RELEASE: True
 
+# scripts to run after deployment
+after_deploy:
+  # build and push our docker image
+  - ps: docker login -u="$env:DOCKER_USER" -p="$env:DOCKER_PASS"
+  - ps: . .\build\build_docker.ps1 "$env:ApVersion"
+
 
 notifications:
   - provider: Email

build/Dockerfile

@@ -0,0 +1,58 @@
+FROM  debian:stretch-slim
+
+ARG GIT_COMMIT
+ARG AP_SOURCE="github"
+ARG AP_VERSION="latest"
+
+LABEL maintainer="Anthony Truskinger <[email protected]>" \
+      description="Debian environment for running AnalysisPrograms.exe" \
+      version="1.0" \
+      org.ecosounds.name="AnalysisPrograms.exe" \
+      org.ecosounds.version=${AP_VERSION} \
+      org.ecosounds.vendor="QUT Ecoacoustics" \
+      org.ecosounds.url="https://github.com/QutEcoacoustics/audio-analysis" \
+      org.ecosounds.vcs-url="https://github.com/QutEcoacoustics/audio-analysis" \
+      org.ecosounds.vcs-ref=${GIT_COMMIT} \
+      org.ecosounds.schema-version="1.0"
+
+
+
+# Install system components (used by powershell. and AP as well)
+RUN apt-get update && apt-get install -y curl gnupg apt-transport-https unzip \
+    readline-common  software-properties-common \
+    wavpack libsox-fmt-all sox shntool libav-tools ffmpeg \
+    # link ffmpeg to /usr/bin/local
+    && ln -s /usr/bin/ffmpeg /usr/local/bin/ffmpeg \
+    && ln -s /usr/bin/ffprobe /usr/local/bin/ffprobe
+
+# install mp3splt
+RUN add-apt-repository "deb http://mp3splt.sourceforge.net/repository unstable main" \
+    && apt-get update --allow-unauthenticated \
+    && apt-get -y --allow-unauthenticated install libmp3splt0-mp3 libmp3splt0-ogg libmp3splt0-flac libmp3splt-doc libmp3splt-dev mp3splt mp3splt-gtk
+
+# Mono
+RUN \
+    # Add mono key server
+    curl https://origin-download.mono-project.com/repo/xamarin.gpg | apt-key add - \
+    # install mono
+    echo "deb http://download.mono-project.com/repo/debian stable-stretch main" | tee /etc/apt/sources.list.d/mono-official-stable.list \
+    && apt-get update \
+    && apt-get install -y mono-complete \
+    && rm -rf /var/lib/apt/lists/* /tmp/*
+
+# Powershell
+RUN \
+     # Import the public repository GPG keys
+    curl https://packages.microsoft.com/keys/microsoft.asc | apt-key add - \
+    # Register the Microsoft Product feed
+    && echo "deb [arch=amd64] https://packages.microsoft.com/repos/microsoft-debian-stretch-prod stretch main" > /etc/apt/sources.list.d/microsoft.list \
+    # Update the list of products
+    && apt-get update \
+    # Install PowerShell
+    && apt-get install -y --no-install-recommends powershell
+
+# Install AP.exe
+ADD download_ap.ps1 /download_ap.ps1
+RUN /usr/bin/pwsh -NonInteractive -c "/download_ap.ps1 ${AP_SOURCE} -version ${AP_VERSION}"
+
+ENTRYPOINT [ "/bin/bash" ]

build/build_docker.ps1

@@ -0,0 +1,27 @@
+#!/usr/bin/pwsh
+
+# builds and pushes a docker file to docker hub
+# currently only builds the 'stable' tag, which is applied to whatever version
+# is supplied to this script.
+# Future work: allow this script to build our 'Weekly' and 'Continuous' lines
+# as their own containers.
+param(
+    # Version tag
+    [Parameter(Mandatory=$true)]
+    [string]
+    $version 
+)
+
+# get the current git commit
+$GIT_COMMIT=git log -1 --format=%H
+
+docker build `
+    -t qutecoacoustics/audio-analysis:stable `
+    -t qutecoacoustics/audio-analysis:latest `
+    -t qutecoacoustics/audio-analysis:$version `
+    . `
+    --build-arg GIT_COMMIT=$GIT_COMMIT `
+    --build-arg AP_SOURCE="github" `
+    --build-arg AP_VERSION=$version
+
+docker push qutecoacoustics/audio-analysis:stable

build/download_ap.ps1

@@ -0,0 +1,118 @@
+#!/usr/bin/pwsh
+
+# Downloads bianry assets for AP.exe
+param(
+    # the source to get the binary from either 'github' or 'appveyor'   
+    [Parameter(Position=0)]
+    [ValidateSet('github','appveyor')]
+    [string]$source,
+
+    # The version to download. Use 'latest' to get the latest version.
+    # Use an actual version number ('18.03.4.1') to get a speicific github version.
+    # Use the AppVeyor build ID ('314') to get a specific appveyor build.
+    [Parameter()]
+    [string]$version = "latest",
+
+    # Which build to get, either 'Debug' or 'Release', defaults to 'Release'
+    [Parameter()]
+    [ValidateSet('Release','Debug')]
+    [string]$build = "Release",
+
+
+    # Directory to extract binary to. Defaults to "/AP" ("C:\AP" on Windows)
+    [Parameter()]
+    [string]$destination = "/AP"
+)
+
+[Net.ServicePointManager]::SecurityProtocol = [Net.SecurityProtocolType]::Tls12
+
+# resolve metadata for asset
+if ($source -eq "github") {
+    $github_url = "https://api.github.com/repos/QutEcoacoustics/audio-analysis/releases"
+    if ($version -eq "latest") {
+        $github_url += "/latest"
+    }
+    else {
+        # strip the leading v if it is present
+        $version = $version -replace "^v",""
+        $github_url += "/tags/v$version"
+    }
+
+    $response = Invoke-RestMethod -Method Get -Uri $github_url
+    $asset_url = $response.assets `
+                 | Where-Object { $_.name -like "$build*" } `
+                 | ForEach-Object browser_download_url
+    Write-Output "Downloading release $($response.tag_name) from GitHub"
+}
+elseif ($source -eq "appveyor") {
+    $appveyor_api = "https://ci.appveyor.com/api"
+    $appveyor_project_url = "$appveyor_api/projects/QUTEcoacousticsResearchGroup/audio-analysis"
+
+    # get the last 50 builds
+    $response = Invoke-RestMethod -Method Get -Uri "$appveyor_project_url/history?recordsNumber=50"
+
+    # filter builds for master branch and build success
+    $ci_builds = $response.builds `
+              | Where-Object { $_.status -eq "success"  -and $_.branch -eq "master" } `
+              | Sort-Object finished -Descending
+    if ($version -eq "latest") {
+        $ci_build = $ci_builds[0]
+    }
+    else {
+        $ci_build = $ci_builds | Where-Object { $_.version -eq $version }
+    }
+
+    if ($null -eq $ci_build) {
+        throw "could not find version '$version' in last 50 AppVeyor builds"
+    }
+
+    # now get the build (we need to do this again because the job sub-object)
+    # is not included in the build objects when the history is retrieved
+    $ci_build = (Invoke-RestMethod -Method Get -Uri "$appveyor_project_url/build/$($ci_build.version)").build
+    $artifacts_url = "$appveyor_api/buildJobs/$($ci_build.jobs[0].jobId)/artifacts"
+    $artifacts = Invoke-RestMethod -Method Get -Uri $artifacts_url
+
+    $file_name =  ($artifacts | Where-Object { $_.fileName -like "*$build*" }).fileName
+
+    $asset_url = "$artifacts_url/$file_name"
+    Write-Output "Downloading version $($ci_build.version) from AppVeyor"
+}
+else {
+    throw "unknown source '$source'"
+}
+
+
+# remove directory if it already exists
+if (Test-Path $destination) {
+    Write-Warning "Deleting old installation at '$destination'"
+    Remove-Item $destination -Recurse -ErrorAction Stop
+    Start-Sleep 1
+}
+New-Item $destination -ItemType Directory -ErrorAction Stop
+
+try {
+    Push-Location $destination
+
+    # download asset using system native curl
+    Write-Output "Downloading asset $asset_url"
+    $curl = Get-Command curl* -CommandType Application
+    & $curl -L -O "$asset_url"
+    if ($LASTEXITCODE -ne 0) {
+        throw "failed downloading $asset_url"
+    }
+
+    $downloaded_zip = Get-ChildItem "$build*.zip"
+
+    unzip -o $downloaded_zip
+    if ($LASTEXITCODE -ne 0) {
+        throw "failed extracting zip $downloaded_zip"
+    }
+
+    Remove-Item $downloaded_zip
+
+    Write-Output "Download complete, installed to $destination"
+
+}
+finally {
+    Pop-Location
+}

src/Acoustics.Shared/Extensions/DoubleExtensions.cs

@@ -48,5 +48,7 @@ public static double RoundToSignficantDigits(this double d, int digits)
             decimal scale = (decimal)Math.Pow(10, Math.Floor(Math.Log10((double)Math.Abs(dec))) + 1);
             return (double)(scale * Math.Round(dec / scale, digits, MidpointRounding.AwayFromZero));
         }
+
+        public static double Floor(this double d) => Math.Floor(d);
     }
 }

src/Acoustics.Shared/ProcessRunner.cs

@@ -405,7 +405,7 @@ private void ProcessTimeout(string arguments, string workingDirectory, int retry
         {
             if (Log.IsVerboseEnabled())
             {
-                Log.Verbose(this.instanceId + ": Process timeout handling code. Exit code: " + this.process.ExitCode);
+                Log.Verbose(this.instanceId + ": Process timeout handling code... Attempting to kill process");
             }
 
             this.process.Refresh();

src/AnalysisPrograms/AcousticWorkbench.Orchestration/EventMetadataResolver.cs

@@ -188,7 +188,9 @@ private async Task<IEnumerable<RemoteSegmentWithData>> DownloadRemoteMetadata(
                 $"Metadata for audio recording media {audioRecordingId} retrieved, generating segments for associated events");
 
             // now generate the segments
-            var limit = audioRecording.DurationSeconds.AsRangeFromZero();
+            // we need to floor the duration to ensure later rounding does round past the limit of the recording
+            Log.Verbose($"Audio recording duration {audioRecording.DurationSeconds} will be floored");
+            var limit = audioRecording.DurationSeconds.Floor().AsRangeFromZero();
             var results = new List<RemoteSegmentWithData>(20);
             foreach (var importedEvent in events)
             {

src/AnalysisPrograms/EventStatistics/ImportedEvent.cs

@@ -10,6 +10,7 @@ namespace AnalysisPrograms.EventStatistics
     using System.Reflection;
     using System.Text;
     using System.Threading.Tasks;
+    using Acoustics.Shared;
     using AcousticWorkbench;
     using CsvHelper.Configuration;
     using log4net;
@@ -44,6 +45,15 @@ public bool IsValid()
                    && this.HighFrequencyHertz.HasValue);
         }
 
+        /// <summary>
+        /// Returns a JSON encoded string that describes this object.
+        /// </summary>
+        /// <returns>The string that describes this object.</returns>
+        public override string ToString()
+        {
+            return Json.SerialiseToString(this);
+        }
+
         public sealed class ImportedEventNameClassMap : CsvClassMap<ImportedEvent>
         {
             private static readonly PropertyInfo[] Properties = typeof(ImportedEvent).GetProperties();

tests/Acoustics.Test/InfiniteTextStreamTests.cs

@@ -62,7 +62,7 @@ void Generate()
 
             Assert.IsTrue(s.Length > 1_000);
 
-            Debug.WriteLine(s);
+            //Debug.WriteLine(s);
         }
 
         [TestMethod]
@@ -103,7 +103,7 @@ void Generate()
 
             Assert.IsTrue(s.Length > 10_000);
 
-            Debug.WriteLine(s);
+            //Debug.WriteLine(s);
         }
     }
 }

tests/Acoustics.Test/Shared/CsvTests.cs

@@ -449,7 +449,9 @@ private void AssertCsvEqual(string expected, FileInfo actual)
             var lines = File.ReadAllText(actual.FullName);
 
             Assert.AreEqual(expected, lines);
-            Debug.WriteLine(lines);
+
+            //Debug.WriteLine(lines);
+
             CollectionAssert.AreEqual(expected.ToArray(), lines.ToArray());
         }