From ed73b8656f5e8d2a644f47dc8b8b32220603dad9 Mon Sep 17 00:00:00 2001 From: kouloumos Date: Fri, 22 Nov 2024 10:21:13 +0200 Subject: [PATCH] feat(scripts): build index - script to build the index - github action to weekly check for new optech topics - update README --- .github/workflows/update-topics.yaml | 67 ++++++++++++ README.md | 61 ++++++++++- scripts/build_index.py | 154 +++++++++++++++++++++++++++ scripts/requirements.txt | 2 + 4 files changed, 282 insertions(+), 2 deletions(-) create mode 100644 .github/workflows/update-topics.yaml create mode 100644 scripts/build_index.py create mode 100644 scripts/requirements.txt diff --git a/.github/workflows/update-topics.yaml b/.github/workflows/update-topics.yaml new file mode 100644 index 0000000..858ae0d --- /dev/null +++ b/.github/workflows/update-topics.yaml @@ -0,0 +1,67 @@ +name: Update Topics Index (Weekly) + +on: + schedule: + - cron: '0 0 * * 0' # Runs at 00:00 UTC every Sunday + workflow_dispatch: # Allows manual triggering + push: + branches: + - 'test/*' # Runs on any branch under test/ + +# Add explicit permissions for the GITHUB_TOKEN +permissions: + contents: write # Allows pushing to the repository + +jobs: + update-topics: + runs-on: ubuntu-latest + + # Add environment variables to control behavior + env: + IS_TEST: ${{ startsWith(github.ref, 'refs/heads/test/') }} + + steps: + - name: Checkout repository + uses: actions/checkout@v4 + + - name: Set up Python + uses: actions/setup-python@v4 + with: + python-version: '3.11' + + - name: Install dependencies + run: pip install -r scripts/requirements.txt + + - name: Build Index + run: python scripts/build_index.py + + - name: Check for changes + id: changes + run: | + git diff --quiet || echo "changes=true" >> $GITHUB_OUTPUT + + - name: Configure Git + if: steps.changes.outputs.changes == 'true' + run: | + git config user.name "GitHub Actions Bot" + git config user.email "actions@github.com" + + # Add debug information in test mode + - name: Debug Info (Test Mode) + if: env.IS_TEST == 'true' + run: | + echo "Changes detected: ${{ steps.changes.outputs.changes }}" + git diff --stat + git status + + - name: Commit and push if there are changes + if: steps.changes.outputs.changes == 'true' + run: | + git add topics_index.json TOPICS.md + # Add [TEST] prefix to commit message on test branches + if [[ "${{ env.IS_TEST }}" == "true" ]]; then + git commit -m "[TEST] Auto-update topics index" + else + git commit -m "Auto-update topics index" + fi + git push \ No newline at end of file diff --git a/README.md b/README.md index 9176fad..951ec6e 100644 --- a/README.md +++ b/README.md @@ -1,2 +1,59 @@ -# topics-index -A list of Bitcoin topics +# Bitcoin Topics Index + +An extensive index of Bitcoin-related topics, combining and enhancing the established [Bitcoin Optech Topics](https://bitcoinops.org/en/topics/) with additional relevant entries. + +See [TOPICS.md](TOPICS.md) for the categorized index or [topics_index.json](topics_index.json) for the machine-readable format. + +## Repository Structure + +``` +. +├── topics/ # Bitcoin topics +│ ├── topic1.yaml +│ ├── topic2.yaml +│ └── ... +├── scripts/ # Build and maintenance scripts +│ └── build_index.py +├── topics_index.json # topics index +├── TOPICS.md # Generated documentation +└── README.md +``` + +## Topics Format + +Each topic is defined in YAML format with the following structure: + +```yaml +title: "Topic Title" # Display name of the topic +slug: "topic-slug" # URL-friendly identifier +categories: # List of categories this topic belongs to + - "Category 1" + - "Category 2" +aliases: # Optional: Alternative names for the topic + - "Alternative Name" + - "Another Name" +excerpt: "A comprehensive description of the topic." +``` + +## Usage + +### Building the Index + +To build the combined index and documentation: + +```bash +python scripts/build_index.py +``` + +This will: + +1. Fetch the latest topics from Bitcoin Optech's [/topics.json](https://bitcoinops.org/topics.json). +2. Combine them with additional topics from the `topics/` directory +3. Generate `topics_index.json` with the complete topics data +4. Create `TOPICS.md` with categorized listings + +### Adding Topics + +1. Create a new YAML file in the `topics/` directory +2. Follow the topics format described above +3. Run the build script to update the index diff --git a/scripts/build_index.py b/scripts/build_index.py new file mode 100644 index 0000000..1ff085c --- /dev/null +++ b/scripts/build_index.py @@ -0,0 +1,154 @@ +import requests +import json +import yaml +import os +import logging +from typing import Dict, List +from collections import defaultdict +from pathlib import Path + +# Set up logging +logging.basicConfig( + level=logging.INFO, format="%(asctime)s - %(levelname)s - %(message)s" +) +logger = logging.getLogger(__name__) + + +class TopicsBuilder: + def __init__(self, optech_topics_url: str, topics_dir: str, root_dir: str): + self.optech_topics_url = optech_topics_url + self.topics_dir = topics_dir + self.root_dir = root_dir + + def fetch_optech_topics(self) -> List[Dict]: + """Fetch topics directly from the Bitcoin Optech website.""" + response = requests.get(self.optech_topics_url) + response.raise_for_status() + return response.json() + + def load_yaml_file(self, filepath: str) -> Dict: + """Load a single YAML file.""" + with open(filepath, "r") as f: + return yaml.safe_load(f) + + def load_topics(self) -> List[Dict]: + """Load all YAML files from the topics directory.""" + topics = [] + if os.path.exists(self.topics_dir): + for filename in os.listdir(self.topics_dir): + if filename.endswith(".yaml"): + filepath = os.path.join(self.topics_dir, filename) + topic = self.load_yaml_file(filepath) + topics.append(topic) + return topics + + def build_topics(self) -> List[Dict]: + """Build complete topics list by combining Optech and additional topics.""" + optech_topics = self.fetch_optech_topics() + additional_topics = self.load_topics() + + # Create a dictionary of topics by slug for easy lookup + topic_dict = {topic["slug"]: topic for topic in optech_topics} + + # Add or override with additional topics + for topic in additional_topics: + topic_dict[topic["slug"]] = topic + + # Convert back to list and sort by title + combined_topics = list(topic_dict.values()) + combined_topics.sort(key=lambda x: x["title"].lower()) + + return combined_topics + + def write_topics_index(self, topics: List[Dict]): + """Write the topics index to a JSON file in the root directory.""" + output_path = os.path.join(self.root_dir, "topics_index.json") + with open(output_path, "w") as f: + json.dump(topics, f, indent=2, ensure_ascii=False) + logger.info(f"Created topics index with {len(topics)} topics") + + def generate_topics_md(self, topics: List[Dict]): + """Generate TOPICS.md documentation file in the root directory.""" + # Group topics by category + categories_dict = defaultdict(list) + unique_topics = set() + + for topic in topics: + topic_categories = topic.get("categories", []) + if isinstance(topic_categories, str): + topic_categories = [topic_categories] + + # Add topic to each of its categories + for category in topic_categories: + # Create topic entry with aliases if they exist + topic_entry = topic["title"] + if "aliases" in topic and topic["aliases"]: + aliases_str = ", ".join(topic["aliases"]) + topic_entry += f" (also covering: {aliases_str})" + + categories_dict[category].append(topic_entry) + unique_topics.add(topic["title"]) + + # Sort categories and their topics + categories = sorted(categories_dict.keys()) + for category in categories: + categories_dict[category].sort() + + # Generate markdown content + content = [] + + # Add summary line + content.append( + f"*{len(categories)} categories for {len(unique_topics)} unique topics, with many appearing in multiple categories.*\n" + ) + + # Add table of contents as a single line + toc_items = [] + for category in categories: + anchor = category.lower().replace(" ", "-") + toc_items.append(f"[{category}](#{anchor})") + content.append(" | ".join(toc_items)) + content.append("") # Empty line after ToC + + # Add categories and their topics + for category in categories: + content.append(f"## {category}") + for topic in categories_dict[category]: + content.append(f"- {topic}") + content.append("") # Empty line between categories + + # Write to file in root directory + output_path = os.path.join(self.root_dir, "TOPICS.md") + with open(output_path, "w") as f: + f.write("\n".join(content)) + logger.info("Created TOPICS.md documentation") + + def build(self): + """Main function to build topics.""" + try: + topics = self.build_topics() + self.write_topics_index(topics) + self.generate_topics_md(topics) + except Exception as e: + logger.error(f"Error during topics building: {str(e)}") + raise + + +def main(): + # Get the absolute path to the repository root (assuming script is in scripts/) + script_dir = Path(__file__).resolve().parent + root_dir = script_dir.parent + + optech_topics_url = "https://bitcoinops.org/topics.json" + topics_dir = os.path.join(root_dir, "topics") + + builder = TopicsBuilder( + optech_topics_url=optech_topics_url, + topics_dir=topics_dir, + root_dir=str(root_dir), + ) + builder.build() + + +if __name__ == "__main__": + main() diff --git a/scripts/requirements.txt b/scripts/requirements.txt new file mode 100644 index 0000000..fda3075 --- /dev/null +++ b/scripts/requirements.txt @@ -0,0 +1,2 @@ +requests +pyyaml \ No newline at end of file