Update JSON files

.github/ISSUE_TEMPLATE/addon_ver.yml

@@ -0,0 +1,40 @@
+name: Addon verification
+description: A request your Meteor Client addon to be added to MeteorAddons.md
+title: "Addon Verification"
+labels: [addon]
+body:
+  - type: input
+    id: name
+    attributes:
+      label: Addon name
+      placeholder: dupe-meteor-addon
+  - type: textarea
+    id: description
+    attributes:
+      label: Description
+      placeholder: An addon for the dupe
+  - type: textarea
+    id: links
+    attributes:
+      label: Links
+      placeholder: github.com/MeteorDevelopment/meteor-client
+      description: Any relevant links to your addon. Eg. repository
+  - type: input
+    id: icon
+    attributes:
+      label: Addon icon url
+      placeholder: github.com/MeteorDevelopment/meteor-addon-template/blob/main/src/main/resources/assets/template/icon.png
+      description: Preferably the same icon used in fabric.mod.json
+  - type: textarea
+    id: authors
+    attributes:
+      label: Authors
+      placeholder: Me
+  - type: checkboxes
+    id: is-open-source
+    attributes:
+      label: Is your addon open-source?
+      description: Check if code for your addon is avaliable on GitHub or any site
+      options:
+        - label: Open-source
+          required: false

.github/workflows/addons.yml

@@ -0,0 +1,62 @@
+name: generate addons.json
+
+concurrency:
+  group: "main"
+  cancel-in-progress: false
+
+on:
+  push:
+    paths:
+      - 'addons.py'
+      - 'verified.json'
+      - 'inject.json'
+      - '.github/workflows/addons.yml'
+  schedule:
+    - cron: "0 5 * * *"
+  workflow_dispatch:
+
+jobs:
+  build:
+    # The type of runner that the job will run on
+    runs-on: ubuntu-latest
+
+    # Steps represent a sequence of tasks that will be executed as part of the job
+    steps:
+      # Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it
+      - uses: actions/checkout@v2
+
+      - name: Set up Python 3.8
+        uses: actions/setup-python@v2
+        with:
+          python-version: '3.8'
+          architecture: 'x64'
+
+      # Cache dependencies. From:
+      # https://github.com/actions/cache/blob/master/examples.md#python---pip
+      - uses: actions/cache@v2
+        with:
+          path: ~/.cache/pip
+          key: ${{ runner.os }}-pip-${{ hashFiles('**/requirements.txt') }}
+          restore-keys: |
+            ${{ runner.os }}-pip-
+      # Install dependencies with `pip`
+      - name: Install requirements
+        run: |
+          python3 -m pip install --upgrade pip setuptools wheel
+          python3 -m pip install -r requirements.txt
+      - name: Generate JSON files
+        run: |
+          python3 --version
+          python3 addons.py
+        env:
+          GH_TOKEN: ${{ secrets.GH_TOKEN }}
+
+      # Commits all changed files to the repository
+      - name: Commit to the repo
+        run: |
+          git config user.name "AntiCope"
+          git config user.email "null"
+          git add .
+          # "echo" returns true so the build succeeds, even if no changed files
+          git commit --amend -m 'Update JSON files' || echo
+          git push --force

README.md

@@ -0,0 +1,5 @@
+<div align="center">
+
+<a href="https://anticope.ml/"><img src="https://user-images.githubusercontent.com/18114966/210327943-c99b23b9-5a85-4040-91bb-e490728bd7ee.png" alt="Visit website"></a>
+
+</div>

addons.py

@@ -0,0 +1,248 @@
+import requests
+import json
+from os import getenv
+from time import sleep
+import re
+from configparser import ConfigParser
+
+
+VERIFIED = json.load(open('verified.json', "r+", encoding='utf-8'))
+INJECT = json.load(open('inject.json', "r+", encoding='utf-8'))
+
+RETRY_COUNT = 25
+
+GH_TOKEN = getenv("GH_TOKEN")
+HEADERS = {"Authorization": f"token {GH_TOKEN}", "Accept": "application/vnd.github.v3+json", "User-Agent": "AntiCope/anticope.ml"}
+
+# regex
+FEATURE_RE = re.compile("(?:add\(new )([^(]+)(?:\([^)]*)\)\)")
+INVITE_RE = re.compile("((?:https?:\/\/)?(?:www.)?(?:discord.(?:gg|io|me|li|com)|discordapp.com\/invite|dsc.gg)\/[a-zA-z0-9-\/]+)")
+MCVER_RE = re.compile("(?:['\"]com\.mojang:minecraft:)([0-9a-z.]+)(?:[\"'])")
+
+def sleep_if_rate_limited(type="search"):
+    for _ in range(RETRY_COUNT):
+        try:
+            r = requests.get("https://api.github.com/rate_limit", headers=HEADERS)
+            if r.status_code != 304 and r.json()['resources'][type]['remaining'] > 0:
+                return
+            print("rate limited. sleeping...")
+        except Exception:
+            print("[rate limit] error. ignoring...")
+        sleep(25)
+
+repos = set(VERIFIED)
+
+# Fetch all repo names that contain meteor entrypoint in fabric.mod.json
+incomplete = True
+page = 0
+print("Fetching based on fabric.mod.json")
+while incomplete:
+    print(f"Fetching page {page}")
+    for _ in range(RETRY_COUNT):
+        try:
+            sleep_if_rate_limited()
+            r = requests.get(
+                f"https://api.github.com/search/code?q=entrypoints+meteor+extension:json+filename:fabric.mod.json+fork:true+in:file&per_page=100&page={page}", headers=HEADERS).json()
+            if 'message' in r.keys() and "rate limit" in r['message']:
+                print("[search fetch] rate limited. sleeping...")
+                sleep(60)
+                continue
+            for file in r['items']:
+                repo = file['repository']
+                if not repo['private']:
+                    repos.add(repo['full_name'])
+            incomplete = r["incomplete_results"]
+            break
+        except Exception:
+            print("[search fetch] error. ignoring...")
+    page += 1
+    if page > 10: # fallback
+        break
+
+# Fetch all repo names that extend MeteorAddon
+incomplete = True
+page = 0
+print("Fetching based on extends MeteorAddon")
+while incomplete:
+    print(f"Fetching page {page}")
+    for _ in range(RETRY_COUNT):
+        try:
+            sleep_if_rate_limited()
+            r = requests.get(
+                f"https://api.github.com/search/code?q=extends+MeteorAddon+language:java+in:file&per_page=100&page={page}", headers=HEADERS).json()
+            if 'message' in r.keys() and "rate limit" in r['message']:
+                print("[search fetch] rate limited. sleeping...")
+                sleep(60)
+                continue
+            for file in r['items']:
+                repo = file['repository']
+                if not repo['private']:
+                    repos.add(repo['full_name'])
+            incomplete = r["incomplete_results"]
+            break
+        except Exception:
+            print("[search fetch] error. ignoring...")
+    page += 1
+    if page > 10: # fallback
+        break
+
+# Request all forks of templates because some people cant click generate
+r = requests.get("https://api.github.com/repos/MeteorDevelopment/meteor-addon-template/forks?per_page=100", headers=HEADERS).json()
+for fork in r:
+    try:
+        repos.add(fork['full_name'])
+    except Exception:
+        print("[fork fetch] error. ignoring...")
+
+# filter templates
+repos = list(filter(lambda x: "-addon-template" not in x.lower(), repos))
+
+def parse_repo(name):
+    sleep_if_rate_limited(type="core")
+    print(f"parsing: {name}")
+
+    repo = requests.get(f"https://api.github.com/repos/{name}", headers=HEADERS).json()
+    fabric = requests.get(f"https://raw.githubusercontent.com/{name}/{repo['default_branch']}/src/main/resources/fabric.mod.json").json()
+
+    # find authors from mod metadata or from github username
+    authors = []
+    for author in fabric['authors']:
+        if type(author) == str:
+            authors.append(author)
+    if len(authors) == 0:
+        authors.append(repo['owner']['login'])
+
+    links = {"github": repo['html_url']}
+
+    if "meteor" not in fabric["entrypoints"].keys():
+        print("Missing meteor entrypoint")
+        raise Exception("Missing meteor entrypoint")
+
+    summary = ""
+    try:
+        summary = repo['description'] or fabric['description']
+    except Exception:
+        print("[summary] error. ignoring...")
+
+    # direct download from releases
+    downloads = 0
+    try:
+        releases = requests.get(f"https://api.github.com/repos/{name}/releases", headers=HEADERS).json()
+        url = None
+        for release in releases:
+            for asset in release['assets']:
+                asset_name: str = asset['name'].lower()
+                if asset_name.endswith("-dev.jar") or asset_name.endswith("-sources.jar"):
+                    continue
+                if asset_name.endswith(".jar"):
+                    url = asset['browser_download_url']
+                    downloads = asset['download_count']
+                    break
+            if url != None:
+                break
+        if url == None:
+            print("missing release")
+        else:
+            links["download"] = url
+    except Exception:
+        print("[dl] error. ignoring...")
+
+    # icon from mod metadata
+    icon = None
+    try:
+        icon = f"https://raw.githubusercontent.com/{name}/{repo['default_branch']}/src/main/resources/{fabric['icon']}"
+        if requests.head(icon).status_code == 404:
+            print("missing icon")
+            icon = None
+    except Exception:
+        print("[icon] error. ignoring...")
+
+    # find discord server by looking at readme mod and repository metadata
+    try:
+        readme = requests.get(f"https://raw.githubusercontent.com/{name}/{repo['default_branch']}/README.md").text
+        invites = INVITE_RE.findall(readme) + INVITE_RE.findall(str(fabric)) + INVITE_RE.findall(str(repo))
+        for invite in invites:
+            if requests.head(invite).status_code != 404:
+                links["discord"] = invite
+                break
+    except Exception:
+        print("[discord invite] error. ignoring...")
+
+    try:
+        site = repo['homepage']
+        if not INVITE_RE.match(site) and site: # skip discord invites
+            links["homepage"] = site
+    except Exception:
+        print("[homepage] error. ignoring...")
+
+    # find features by parsing the entrypoint
+    features = []
+    feature_count = 0
+    try:
+        entrypoint = requests.get(f"https://raw.githubusercontent.com/{name}/{repo['default_branch']}/src/main/java/{fabric['entrypoints']['meteor'][0].replace('.', '/')}.java").text
+        features.extend([str(x) for x in FEATURE_RE.findall(entrypoint)])
+        feature_count = len(features)
+        if len(features) > 50:
+            count = len(features) - 50
+            features = features[:50]
+            features.append(f"...and {count} more")
+    except Exception:
+        print("[features] error. ignoring...")
+
+    # parse build.gradle
+    mc_version = None
+    try:
+        build_gradle = requests.get(f"https://raw.githubusercontent.com/{name}/{repo['default_branch']}/build.gradle").text
+        try:
+            props = requests.get(f"https://raw.githubusercontent.com/{name}/{repo['default_branch']}/gradle.properties").text
+            props = "[conf]\n"+props # convert to ini format
+            gradle_props = ConfigParser()
+            gradle_props.read_string(props)
+            for key, val in dict(gradle_props['conf']).items():
+                build_gradle = build_gradle.replace("${project."+key+"}", val)
+                build_gradle = build_gradle.replace(f"$project.{key}", val)
+                build_gradle = build_gradle.replace(f"project.{key}", val)
+        except Exception as ex:
+            print(f"[build.gradle] failed to read gradle.properties.")  
+        mc_version = MCVER_RE.findall(build_gradle)[0]
+    except Exception:
+        print("[build.gradle] error. ignoring...")  
+
+    result = {
+        "authors": authors,
+        "features": features,
+        "feature_count": feature_count,
+        "icon": icon,
+        "id": repo['full_name'],
+        "links": links,
+        "name": fabric['name'],
+        "stars": repo['stargazers_count'],
+        "last_update": repo['pushed_at'],
+        "downloads": downloads,
+        "mc_version": mc_version,
+        "status": {
+            "archived": repo['archived']
+        },
+        "verified": (repo['full_name'] in VERIFIED),
+        "summary": summary
+    }
+
+    result.update(INJECT.get(repo['full_name'], {}))
+    return result
+
+verified_json = []
+unverified_json = []
+for repo in repos:
+    if repo in VERIFIED:
+        try:
+            verified_json.append(parse_repo(repo))
+        except Exception as ex:
+            print(f"error {ex}. ignoring..., repo: {repo}")
+    else:
+        try:
+            unverified_json.append(parse_repo(repo))
+        except Exception as ex:
+            print(f"error {ex}. ignoring..., repo: {repo}")
+
+json.dump(verified_json, open("addons-ver.json", "w+", encoding='utf-8'), indent=None)
+json.dump(unverified_json, open("addons-unver.json", "w+", encoding='utf-8'), indent=None)

inject.json

@@ -0,0 +1,12 @@
+{
+    "Declipsonator/Troll-Addon": {
+        "summary": "No rat is present in this build.",
+        "authors": [ "Declipsonator" ]
+    },
+    "cally72jhb/vector-addon": {
+        "icon": "https://user-images.githubusercontent.com/18114966/159967693-d42e1b63-2fb2-4261-9913-9c1b95c43f69.png"
+    },
+    "AntiCope/reaper-addon": {
+        "icon": "https://raw.githubusercontent.com/AntiCope/reaper-addon/main/src/main/resources/assets/reaper/icon.png"
+    }
+}

requirements.txt

@@ -0,0 +1 @@
+requests