From a89d92e2e84dcca8e20b7bef20e773c5c953f9fa Mon Sep 17 00:00:00 2001 From: Matt Beckett Date: Wed, 26 Jul 2023 21:50:57 +0100 Subject: [PATCH 1/8] Add basic Dockerfile for building fixed version of Taky --- Dockerfile | 27 +++++++++++++++++++++++++++ 1 file changed, 27 insertions(+) create mode 100644 Dockerfile diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..710d2a6 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,27 @@ +# First stage: builder +FROM python:3.11 as builder + +ENV TAKY_VERSION=0.9 + +WORKDIR /build + +RUN git clone --depth 1 https://github.com/tkuester/taky.git -b ${TAKY_VERSION} + +WORKDIR /build/taky + +RUN python3 -m pip install --upgrade pip && \ + python3 -m pip install -r requirements.txt && \ + python3 setup.py install && \ + find /usr/local -name '*.pyc' -delete && \ + find /usr/local -name '__pycache__' -type d -exec rm -rf {} + + +# Second stage: runtime +FROM python:3.11-slim as runtime + +WORKDIR / + +COPY --from=builder /usr/local /usr/local + +RUN mkdir -p /var/taky + +ENTRYPOINT [ "taky", "-c", "/taky/taky.conf" ] From 0473f5134125b6580a0911d8fdee05c1076a9b02 Mon Sep 17 00:00:00 2001 From: Matt Beckett Date: Thu, 27 Jul 2023 15:39:44 +0100 Subject: [PATCH 2/8] Add GitHub action workflow for building Taky images --- .github/workflows/build-docker.yaml | 68 +++++++++++++++++++++++++++++ 1 file changed, 68 insertions(+) create mode 100644 .github/workflows/build-docker.yaml diff --git a/.github/workflows/build-docker.yaml b/.github/workflows/build-docker.yaml new file mode 100644 index 0000000..460a122 --- /dev/null +++ b/.github/workflows/build-docker.yaml @@ -0,0 +1,68 @@ +on: + pull_request: + push: + branches: + - 'main' + +env: + PLATFORMS: linux/amd64, linux/arm64, linux/armv7 + +jobs: + build_images: + runs-on: ubuntu-latest + permissions: + contents: read + id-token: write + + steps: + - name: Checkout + uses: actions/checkout@v2 + + - name: Cache Docker layers + uses: actions/cache@v2 + with: + path: /tmp/.buildx-cache + key: ${{ runner.os }}-buildx-${{ inputs.image-name }} + restore-keys: | + ${{ runner.os }}-buildx- + + - name: Set up QEMU + # Needed as part of https://github.com/docker/buildx/issues/495#issuecomment-1043341496 + id: qemu + uses: docker/setup-qemu-action@v1 + with: + image: tonistiigi/binfmt:latest + platforms: all + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v2 + + - name: Login to GitHub Container Registry + uses: docker/login-action@v2 + with: + registry: ghcr.io + username: ${{ github.repository_owner }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Docker meta for postgres-operator + id: postgres-operator-meta + uses: docker/metadata-action@v4 + with: + images: ghcr.io/${{ github.repository_owner }}/taky + + - name: Build and push + uses: docker/build-push-action@v3 + with: + context: docker/${{inputs.image-name}} + push: ${{inputs.push}} + platforms: ${{ env.PLATFORMS }} + pull: true + build-args: VERSION=${{ github.ref_name }} + tags: ${{ steps.docker-meta.outputs.tags }} + cache-from: type=local,src=/tmp/.buildx-cache + cache-to: type=local,dest=/tmp/.buildx-cache-new + + - name: Move cache + run: | + rm -rf /tmp/.buildx-cache + mv /tmp/.buildx-cache-new /tmp/.buildx-cache From eb1e35bd55b68aa5988dcc5fb9bc5fefdc052d15 Mon Sep 17 00:00:00 2001 From: Matt Beckett Date: Thu, 27 Jul 2023 15:42:55 +0100 Subject: [PATCH 3/8] Fix up invalid context and left-over invalid image name --- .github/workflows/build-docker.yaml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/build-docker.yaml b/.github/workflows/build-docker.yaml index 460a122..e678275 100644 --- a/.github/workflows/build-docker.yaml +++ b/.github/workflows/build-docker.yaml @@ -44,8 +44,8 @@ jobs: username: ${{ github.repository_owner }} password: ${{ secrets.GITHUB_TOKEN }} - - name: Docker meta for postgres-operator - id: postgres-operator-meta + - name: Docker meta for taky + id: taky-meta uses: docker/metadata-action@v4 with: images: ghcr.io/${{ github.repository_owner }}/taky @@ -53,7 +53,7 @@ jobs: - name: Build and push uses: docker/build-push-action@v3 with: - context: docker/${{inputs.image-name}} + context: . push: ${{inputs.push}} platforms: ${{ env.PLATFORMS }} pull: true From 7dbf574de82609c1a9104edba2182aef4d100e22 Mon Sep 17 00:00:00 2001 From: Matt Beckett Date: Thu, 27 Jul 2023 16:21:18 +0100 Subject: [PATCH 4/8] Change push from input to true --- .github/workflows/build-docker.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build-docker.yaml b/.github/workflows/build-docker.yaml index e678275..b84bcaf 100644 --- a/.github/workflows/build-docker.yaml +++ b/.github/workflows/build-docker.yaml @@ -54,7 +54,7 @@ jobs: uses: docker/build-push-action@v3 with: context: . - push: ${{inputs.push}} + push: true platforms: ${{ env.PLATFORMS }} pull: true build-args: VERSION=${{ github.ref_name }} From 6b410b96ee9dcbe5239fa9f30e5d95e89fa59e2d Mon Sep 17 00:00:00 2001 From: Matt Beckett Date: Thu, 27 Jul 2023 16:24:33 +0100 Subject: [PATCH 5/8] Fix invalid reference to docker metadata tags --- .github/workflows/build-docker.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build-docker.yaml b/.github/workflows/build-docker.yaml index b84bcaf..363d7ed 100644 --- a/.github/workflows/build-docker.yaml +++ b/.github/workflows/build-docker.yaml @@ -58,7 +58,7 @@ jobs: platforms: ${{ env.PLATFORMS }} pull: true build-args: VERSION=${{ github.ref_name }} - tags: ${{ steps.docker-meta.outputs.tags }} + tags: ${{ steps.taky-meta.outputs.tags }} cache-from: type=local,src=/tmp/.buildx-cache cache-to: type=local,dest=/tmp/.buildx-cache-new From 1058cbee05d74da17a96d612008fac848004816a Mon Sep 17 00:00:00 2001 From: Matt Beckett Date: Tue, 8 Aug 2023 00:02:01 +0100 Subject: [PATCH 6/8] Fix uo dockerfile --- Dockerfile | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/Dockerfile b/Dockerfile index 710d2a6..5d39e2e 100644 --- a/Dockerfile +++ b/Dockerfile @@ -2,6 +2,7 @@ FROM python:3.11 as builder ENV TAKY_VERSION=0.9 +ENV PUBLIC_IP=192.168.0.60 WORKDIR /build @@ -15,13 +16,16 @@ RUN python3 -m pip install --upgrade pip && \ find /usr/local -name '*.pyc' -delete && \ find /usr/local -name '__pycache__' -type d -exec rm -rf {} + +RUN takyctl setup --public-ip=${PUBLIC_IP} /etc/taky + # Second stage: runtime FROM python:3.11-slim as runtime WORKDIR / -COPY --from=builder /usr/local /usr/local +RUN mkdir /var/taky -RUN mkdir -p /var/taky +COPY --from=builder /usr/local /usr/local +COPY --from=builder /etc/taky /etc/taky -ENTRYPOINT [ "taky", "-c", "/taky/taky.conf" ] +ENTRYPOINT [ "taky", "-c", "/etc/taky/taky.conf" ] From 691f69fb7fa69a0988f2d54280b1b1e0598ad4de Mon Sep 17 00:00:00 2001 From: Matt Beckett Date: Tue, 8 Aug 2023 00:02:07 +0100 Subject: [PATCH 7/8] Update .gitignore --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index 4ef7195..85d5c3e 100644 --- a/.gitignore +++ b/.gitignore @@ -18,3 +18,4 @@ htmlcov *.pem *.p12 *.zip +.venv From c94f42762f71a838cf1fe5b1ec0e0715b9b4d55a Mon Sep 17 00:00:00 2001 From: Matt Beckett Date: Tue, 8 Aug 2023 00:03:19 +0100 Subject: [PATCH 8/8] Add basic endpoint for uploading datapackage to dps via itak client --- taky/dps/views/datapackage.py | 53 +++++++++++++++++++++++++++++++++++ 1 file changed, 53 insertions(+) diff --git a/taky/dps/views/datapackage.py b/taky/dps/views/datapackage.py index b2d98c5..ff7147a 100644 --- a/taky/dps/views/datapackage.py +++ b/taky/dps/views/datapackage.py @@ -1,5 +1,7 @@ import os import json +import hashlib +import magic from datetime import datetime as dt from flask import request, send_file @@ -99,6 +101,57 @@ def datapackage_get(): return send_file(name, as_attachment=True, download_name=meta["Name"]) +#Experimental reverse-engineered endpoint for iTAK Datapackage upload +@app.route("/Marti/sync/upload", methods=["POST"]) +@requires_auth +def datapackage_upload_itak(): + mime = magic.Magic(mime=True) + try: + name = request.args["name"] + uid = request.args["uid"] + creator_uid = request.args["CreatorUid"] + f_hash = hashlib.sha256(request.data) + keywords = request.args["keywords"] + except KeyError: + return "Invalid arguments", 400 + + filename = secure_filename(f"{creator_uid}_{name}") + + meta = get_meta(f_name=filename) + if meta.get("Hash") != f_hash: + old_meta_hash_path = os.path.join( + app.config["UPLOAD_PATH"], "meta", f'{meta.get("Hash")}.json' + ) + try: + os.unlink(old_meta_hash_path) + except: # pylint: disable=bare-except + pass + + # Save the uploaded file + file_path = os.path.join(app.config["UPLOAD_PATH"], filename) + with open(file_path, "wb") as binary_file: + binary_file.write(request.data) + + sub_user = request.headers.get("X-USER", "Anonymous") + meta = { + "UID": uid, # What the file will be saved as + "Name": name, # File name on the server + "Hash": f_hash.hexdigest(), # SHA-256, checked + "PrimaryKey": 1, # Not used, must be >= 0 + "SubmissionDateTime": dt.utcnow().isoformat() + "Z", + "SubmissionUser": sub_user, + "CreatorUid": creator_uid, + "Keywords": f"{keywords}", + "MIMEType": f"{mime.from_buffer(request.data)}", + "Size": os.path.getsize(file_path), # Checked, do not fake + "Visibility": "public", + } + + put_meta(meta) + + # src/main/java/com/atakmap/android/missionpackage/http/MissionPackageDownloader.java:539 + # This is needed for client-to-client data package transmission + return url_for(f_hash.hexdigest()) @app.route("/Marti/sync/missionupload", methods=["POST"]) @requires_auth