Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

💚 update the CI actions and fix publish-packages.yml workflow #108

Merged
merged 3 commits into from
Apr 5, 2024
Merged

💚 update the CI actions and fix publish-packages.yml workflow #108

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
607746b
:green_heart: work on CI
AucaCoyan Apr 5, 2024
6c48657
:art: format python
AucaCoyan Apr 5, 2024
26c90c0
:green_heart: fix missing email in CI git CLI
AucaCoyan Apr 5, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
239 changes: 133 additions & 106 deletions .github/scripts/publish_packages.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -9,131 +9,158 @@
import time
import json
import re
from typing import TypedDict
from typing import List
from typing import Tuple
from typing import Tuple


Package = namedtuple('Package', 'name version location title description author tags')
UploadedPackage = namedtuple('UploadedPackage', 'name version')
Package = namedtuple("Package", "name version location title description author tags")
UploadedPackage = namedtuple("UploadedPackage", "name version")

should_publish = os.getenv("PUBLISH", "false") == "true"

RELEASES_VERSION = "v1.0.0"


# Get the packages that have been published on the Github Releases section
def get_released_packages() -> List[UploadedPackage]:
result = subprocess.run(['gh', 'release', 'view', "--json", "assets"], stdout=subprocess.PIPE)
json_assets = result.stdout.decode('utf-8')
assets = json.loads(json_assets)["assets"]
result = subprocess.run(
["gh", "release", "view", "--json", "assets"], stdout=subprocess.PIPE
)
json_assets = result.stdout.decode("utf-8")
assets = json.loads(json_assets)["assets"]

packages = []
for package in assets:
file_name = package["name"]
match = re.search("(?P<name>.*?)-(?P<version>\d+.\d+.\d+).zip", file_name)
if match is not None:
name = match.group("name")
version = match.group("version")

packages = []
for package in assets:
file_name = package["name"]
match = re.search('(?P<name>.*?)-(?P<version>\d+.\d+.\d+).zip', file_name)
if match is not None:
name = match.group("name")
version = match.group("version")
packages.append(UploadedPackage(name, version))

packages.append(UploadedPackage(name, version))
return packages

return packages

# Get the packages defined in the hub repository
def get_repository_packages() -> List[Package]:
packages = []
for path in glob.glob("packages/*/*/_manifest.yml"):
package_dir = os.path.dirname(path)
with open(path, 'r') as stream:
try:
manifest = yaml.safe_load(stream)
name = manifest["name"]
version = manifest["version"]
title = manifest["title"]
description = manifest["description"]
author = manifest["author"]
tags = manifest.get("tags", [])
package = Package(name, version, package_dir, title, description, author, tags)
packages.append(package)
except yaml.YAMLError as exc:
print("Exception parsing YAML ", exc)
return packages

# Calculate which packages have NOT been published yet on Releases,
packages = []
for path in glob.glob("packages/*/*/_manifest.yml"):
package_dir = os.path.dirname(path)
with open(path, "r") as stream:
try:
manifest = yaml.safe_load(stream)
name = manifest["name"]
version = manifest["version"]
title = manifest["title"]
description = manifest["description"]
author = manifest["author"]
tags = manifest.get("tags", [])
package = Package(
name, version, package_dir, title, description, author, tags
)
packages.append(package)
except yaml.YAMLError as exc:
print("Exception parsing YAML ", exc)
return packages


# Calculate which packages have NOT been published yet on Releases,
# the ones that will need to be published during this run
def calculate_missing_packages(released: List[Package], repository: List[Package]) -> List[Package]:
published_packages: set[str] = set()
for package in released:
published_packages.add(f"{package.name}@{package.version}")

missing_packages = []
for package in repository:
package_id = f"{package.name}@{package.version}"
if package_id not in published_packages:
missing_packages.append(package)

return missing_packages

# Calculate SHA256 of the given file
def calculate_missing_packages(
released: List[Package], repository: List[Package]
) -> List[Package]:
published_packages: set[str] = set()
for package in released:
published_packages.add(f"{package.name}@{package.version}")

missing_packages = []
for package in repository:
package_id = f"{package.name}@{package.version}"
if package_id not in published_packages:
missing_packages.append(package)

return missing_packages


# Calculate SHA256 of the given file
def calculate_sha256(filename) -> str:
sha256_hash = hashlib.sha256()
with open(filename,"rb") as f:
for byte_block in iter(lambda: f.read(4096),b""):
sha256_hash.update(byte_block)
return sha256_hash.hexdigest()
sha256_hash = hashlib.sha256()
with open(filename, "rb") as f:
for byte_block in iter(lambda: f.read(4096), b""):
sha256_hash.update(byte_block)
return sha256_hash.hexdigest()


# Archive the given package, calculating its hash
def create_archive(package: Package)-> Tuple[str, str, str]:
temp_dir = tempfile.gettempdir()
target_name = os.path.join(temp_dir, f"{package.name}-{package.version}")
shutil.make_archive(target_name, 'zip', package.location)

target_file = f"{target_name}.zip"
hash = calculate_sha256(target_file)
target_sha_file = os.path.join(temp_dir, f"{package.name}-{package.version}-sha256.txt")
def create_archive(package: Package) -> Tuple[str, str, str]:
temp_dir = tempfile.gettempdir()
target_name = os.path.join(temp_dir, f"{package.name}-{package.version}")
shutil.make_archive(target_name, "zip", package.location)

target_file = f"{target_name}.zip"
hash = calculate_sha256(target_file)
target_sha_file = os.path.join(
temp_dir, f"{package.name}-{package.version}-sha256.txt"
)

with open(target_sha_file, "w") as file:
file.write(hash)
return (target_file, target_sha_file, hash)
with open(target_sha_file, "w") as file:
file.write(hash)

return (target_file, target_sha_file, hash)


# Upload the package files (archive + SHA256 sum) on GH Releases
def upload_to_releases(archive_path, archive_hash_path):
if should_publish:
subprocess.run(["gh", "release", "upload", RELEASES_VERSION, archive_hash_path, archive_path, "--clobber" ])
if should_publish:
subprocess.run(
[
"gh",
"release",
"upload",
RELEASES_VERSION,
archive_hash_path,
archive_path,
"--clobber",
]
)


# Generate the updated index and upload it to GH Releases
def update_index(repository: List[Package]):
packages = []

for package in repository:
packages.append({
"name": package.name,
"author": package.author,
"description": package.description,
"title": package.title,
"version": package.version,
"tags": package.tags,
"archive_url": f"https://github.com/espanso/hub/releases/latest/download/{package.name}-{package.version}.zip",
"archive_sha256_url": f"https://github.com/espanso/hub/releases/latest/download/{package.name}-{package.version}-sha256.txt",
})

index = {
"last_update": round(time.time()),
"packages": packages,
}

json_index = json.dumps(index)

temp_dir = tempfile.gettempdir()
target_file = os.path.join(temp_dir, "package_index.json")

with open(target_file, "w") as file:
file.write(json_index)

if should_publish:
subprocess.run(["gh", "release", "upload", RELEASES_VERSION, target_file, "--clobber"])
packages = []

for package in repository:
packages.append(
{
"name": package.name,
"author": package.author,
"description": package.description,
"title": package.title,
"version": package.version,
"tags": package.tags,
"archive_url": f"https://github.com/espanso/hub/releases/latest/download/{package.name}-{package.version}.zip",
"archive_sha256_url": f"https://github.com/espanso/hub/releases/latest/download/{package.name}-{package.version}-sha256.txt",
}
)

index = {
"last_update": round(time.time()),
"packages": packages,
}

json_index = json.dumps(index)

temp_dir = tempfile.gettempdir()
target_file = os.path.join(temp_dir, "package_index.json")

with open(target_file, "w") as file:
file.write(json_index)

if should_publish:
subprocess.run(
["gh", "release", "upload", RELEASES_VERSION, target_file, "--clobber"]
)


print("Reading packages from repository...")
Expand All @@ -147,24 +174,24 @@ def update_index(repository: List[Package]):
missing_packages = calculate_missing_packages(released_packages, repository_packages)

if len(missing_packages) == 0:
print("Packages are already up-to-date")
quit(0)
print("Packages are already up-to-date")
quit(0)

print("")
print("Packages to publish:")
for package in missing_packages:
print(f"--> {package.name}@{package.version}")
print(f"--> {package.name}@{package.version}")

archive_path, archive_hash_path, archive_hash = create_archive(package)
archive_path, archive_hash_path, archive_hash = create_archive(package)

print(f"Created archive {archive_path}, hash: {archive_hash}")
print(f"Created archive {archive_path}, hash: {archive_hash}")

print(f"Uploading to Github Releases...")
upload_to_releases(archive_path, archive_hash_path)
print("Done!")
print("Uploading to Github Releases...")
upload_to_releases(archive_path, archive_hash_path)
print("Done!")

print("")
print("Updating index...")
update_index(repository_packages)

print("Done!")
print("Done!")
78 changes: 43 additions & 35 deletions .github/scripts/validate/main.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6,49 +6,57 @@
report_errors = []

for package in glob.glob("packages/*"):
package_name = os.path.basename(package)
package_name = os.path.basename(package)

print(f"Validating package: {package_name}... ", end='', flush=True)
print(f"Validating package: {package_name}... ", end="", flush=True)

errors = validate_package(package)
if len(errors) == 0:
print("OK")
continue
errors = validate_package(package)
if len(errors) == 0:
print("OK")
continue

print(f"Found {len(errors)} errors:")
print(f"Found {len(errors)} errors:")

for error in errors:
print(f"Check: {error.name}")
print(f" ->: {error.error}")

report_errors.append({
"package": package_name,
"errors": errors,
})
for error in errors:
print(f"Check: {error.name}")
print(f" ->: {error.error}")

report_errors.append(
{
"package": package_name,
"errors": errors,
}
)

print("Generating CI report...")

with open("validation_report.md", "w", encoding="utf-8") as f:
f.write("## CI Quality Check 🤖🚨 \n\n")

if len(report_errors) == 0:
f.write("All checks passed ✅ Great job!\n\n")
f.write("![Great Job](https://raw.githubusercontent.com/espanso/hub/main/.github/scripts/validate/great-job-image.jpg)")
else:
f.write("Oh snap! Our robots detected some errors 🤖 We need to solve them before merging the package:\n\n")
for package in report_errors:
package_name = package["package"]
package_errors = package["errors"]
f.write(f"### Package: {package_name}\n\n")
for error in package_errors:
error_name = error.name
error_message = error.error
f.write(f"#### Check: **{error_name}** ❌\n\n")
f.write(f"```\n{error_message}\n```\n\n")
f.write("After you fixed the problems, please create another commit and push it to re-run the checks 🚀")
f.write("## CI Quality Check 🤖🚨 \n\n")

if len(report_errors) == 0:
f.write("All checks passed ✅ Great job!\n\n")
f.write(
"![Great Job](https://raw.githubusercontent.com/espanso/hub/main/.github/scripts/validate/great-job-image.jpg)"
)
else:
f.write(
"Oh snap! Our robots detected some errors 🤖 We need to solve them before merging the package:\n\n"
)
for package in report_errors:
package_name = package["package"]
package_errors = package["errors"]
f.write(f"### Package: {package_name}\n\n")
for error in package_errors:
error_name = error.name
error_message = error.error
f.write(f"#### Check: **{error_name}** ❌\n\n")
f.write(f"```\n{error_message}\n```\n\n")
f.write(
"After you fixed the problems, please create another commit and push it to re-run the checks 🚀"
)

if len(report_errors) == 0:
print("All ok!")
print("All ok!")
else:
print("Errors detected, please see attached report.")
sys.exit(1)
print("Errors detected, please see attached report.")
sys.exit(1)
Loading