Pipeline cleanup

.bb/dependencies.txt

@@ -1 +1 @@
-https://github.com/biobricks-ai/pubchem-annotations#4bc6de375ba7dbdbd2986b53d952e41ace592551
+https://github.com/biobricks-ai/pubchem-annotations#3b1b417e43b17b6caf75a730f8c4119aa76f8c16

.gitignore

@@ -2,3 +2,4 @@ logs
 /download
 /list
 /brick
+/cache

dvc.yaml

@@ -1,11 +1,21 @@
 stages:
 
   update_biobricks_dependencies:
-    cmd: stages/00_update_biobricks_dependencies.py
+    cmd: stages/00_update_biobricks_deps.sh
+    deps:
+      - stages/00_update_biobricks_deps.sh
 
   process-annotations:
     cmd: stages/01_process.py
     deps:
       - .bb/dependencies.txt
+      - stages/01_process.py
+    outs:
+      - cache/process/combined_annotations.ttl
+  build:
+    cmd: stages/02_build.sh
+    deps:
+      - cache/process/combined_annotations.ttl
+      - stages/02_build.sh
     outs:
-      - brick/pubchem-annotations.hdt
+      - brick/annotations.hdt

stages/00_update_biobricks_deps.sh

@@ -0,0 +1,3 @@
+#!/usr/bin/bash
+
+biobricks pull

stages/01_process.py

@@ -1,3 +1,5 @@
+#!/usr/bin/env python3
+
 import biobricks as bb
 import pandas as pd
 import pyarrow.parquet as pq
@@ -148,34 +150,3 @@
         with open(file, "r") as infile:
             outfile.write(infile.read())
             outfile.write("\n")  # Ensure separation between files
-
-print("Creating HDT file ...")
-# Convert the Turtle file to an HDT file
-hdt_file = str(outdir / 'annotations.hdt')
-# # # create empty HDT file
-# with open(hdt_file, "w") as f:
-#     pass
-
-# # # Create an HDT store
-# store = HDTStore(hdt_file)
-
-# # # load the entire graph into the store
-# g = Graph()
-# g.parse(combined_file, format='ttl')
-
-# # # add the graph to the store
-# store.load_graph(g) # or add_graph?
-
-# # close the store to finalize the HDT file
-# store.close()
-
-# hdt = HDTDocument().from_graph(g)
-
-# Conversion using the command-line tool rdf2hdtcat
-subprocess.run(["rdf2hdtcat -p", combined_file, hdt_file], check=True)
-# # Conversion using the command-line tool rdf2hdt
-# subprocess.run(["rdf2hdt -p", combined_file, hdt_file], check=True)
-print(f"Done writing HDT file to {hdt_file}")
-
-# # delete cache directory
-# shutil.rmtree(pathlib.Path('cache'))

stages/02_build.sh

@@ -0,0 +1,20 @@
+#!/usr/bin/env bash
+
+set -euo pipefail
+
+# Get local path
+localpath=$(pwd)
+echo "Local path: $localpath"
+
+# Get cache path
+cachepath="$localpath/cache"
+echo "Cache path: $cachepath"
+
+# Create brick directory
+brickpath="$localpath/brick"
+mkdir -p $brickpath
+echo "Brick path: $brickpath"
+
+export base_uri="http://rdf.ncbi.nlm.nih.gov/pubchem/annotations.hdt"
+
+rdf2hdt -i -p -B "$base_uri"  $cachepath/process/combined_annotations.ttl $brickpath/annotations.hdt

stages/02_test.py

@@ -1,3 +1,5 @@
+#!/usr/bin/env python3
+
 import rdflib
 import pathlib
 from rdflib_hdt import HDTStore