From 3a746019e2389554f6a2c368f051e782d38a8ae7 Mon Sep 17 00:00:00 2001 From: Teal Dulcet Date: Thu, 26 Sep 2024 04:01:28 -0700 Subject: [PATCH] Updated PrimeNet program and README. --- README.md | 92 ++++++------ cudalucas.sh | 2 +- cudalucas2.sh | 2 +- gpuowl.sh | 4 +- gpuowl2.sh | 4 +- mlucas.sh | 2 +- mprime-python-port/mprime.py | 7 + primenet.py | 278 +++++++++++++++++++---------------- 8 files changed, 214 insertions(+), 177 deletions(-) diff --git a/README.md b/README.md index 3e31b41..c4219fe 100644 --- a/README.md +++ b/README.md @@ -48,7 +48,7 @@ wget -qO - https://raw.github.com/tdulcet/Distributed-Computing-Scripts/master/g ### PrimeNet -Automatically gets assignments, reports assignment results, upload proof files and optionally registers assignments and reports assignment progress to PrimeNet for the GpuOwl, CUDALucas, Mlucas, mfaktc, and mfakto GIMPS programs. Supports both Python 2 and 3 and Windows, macOS and Linux. Requires the [Requests library](https://requests.readthedocs.io/en/master/), which is included with most Python 3 installations. The script will automatically install Requests on first run if it is not already installed. GIMPS [discontinued first time LL assignments](https://mersenneforum.org/showthread.php?t=26682) in April 2021, although the script [still supports them](https://mersenneforum.org/showthread.php?p=575260#post575260) for users of CUDALucas or with limited disk space. Our [GpuOwl](#gpuowl), [CUDALucas](#cudalucas) and [Mlucas](#mlucas) Linux scripts automatically download, setup and run this. Adapted from the PrimeNet Python script from [Mlucas](https://www.mersenneforum.org/mayer/README.html#download2) by [Loïc Le Loarer](https://github.com/llloic11/primenet) and Ernst W. Mayer, which itself was adapted from primetools by [Mark Rose](https://github.com/MarkRose/primetools) and [teknohog](https://github.com/teknohog/primetools). +Automatically gets and registers assignments, reports assignment progress and results, uploads proof files to and downloads certification starting values from PrimeNet for the Mlucas, GpuOwl/PRPLL, CUDALucas, mfaktc and mfakto GIMPS programs. Additionally, it can get assignments and report results to mersenne.ca for exponents above the PrimeNet limit of 1G. Supports both Python 2 and 3 and Windows, macOS and Linux. Requires the [Requests library](https://requests.readthedocs.io/en/latest/), which is included with most Python 3 installations. The program will automatically prompt to install Requests on first run if it is not already installed. GIMPS [discontinued first time LL assignments](https://mersenneforum.org/showthread.php?t=26682) in April 2021, although the program [still supports them](https://mersenneforum.org/showthread.php?p=575260#post575260) for users of CUDALucas or with limited disk space. Our [GpuOwl](#gpuowl), [CUDALucas](#cudalucas) and [Mlucas](#mlucas) Linux scripts automatically download, setup and run this. Adapted from the PrimeNet Python script from [Mlucas](https://www.mersenneforum.org/mayer/README.html#download2) by [Loïc Le Loarer](https://github.com/llloic11/primenet) and Ernst W. Mayer, which itself was adapted from primetools by [Mark Rose](https://github.com/MarkRose/primetools) and [teknohog](https://github.com/teknohog/primetools). #### Usage @@ -57,16 +57,17 @@ Usage: primenet.py [options] Use -h/--help to see all options Use --setup to configure this instance of the program -This program will automatically get assignments, report assignment results, -upload proof files and optionally register assignments and report assignment -progress to PrimeNet for the Mlucas, GpuOwl/PRPLL, CUDALucas, mfaktc and -mfakto GIMPS programs. It also saves its configuration to a “local.ini” file -by default, so it is only necessary to give most of the arguments once. The -first time it is run, if a password is NOT provided, it will register the -current Mlucas/GpuOwl/PRPLL/CUDALucas/mfaktc/mfakto instance with PrimeNet -(see the Registering Options below). Then, it will report assignment results, -get assignments and upload any proof files to PrimeNet on the --timeout -interval, or only once if --timeout is 0. If registered, it will additionally +This program will automatically get and register assignments, report +assignment progress and results, upload proof files to and download +certification starting values from PrimeNet for the Mlucas, GpuOwl/PRPLL, +CUDALucas, mfaktc and mfakto GIMPS programs. It can get assignments and report +results to mersenne.ca for exponents above the PrimeNet limit of 1G. It also +saves its configuration to a 'local.ini' file by default, so it is only +necessary to give most of the arguments once. The first time it is run, it +will register the current Mlucas/GpuOwl/PRPLL/CUDALucas/mfaktc/mfakto instance +with PrimeNet (see the Registering Options below). Then, it will report +assignment results, get assignments and upload any proof files to PrimeNet on +the --timeout interval, or only once if --timeout is 0. It will additionally report the progress on the --checkin interval. Options: @@ -82,15 +83,15 @@ Options: each worker. It automatically sets the --cpu-num option for each directory. -i WORKTODO_FILE, --work-file=WORKTODO_FILE - Work file filename, Default: “worktodo.txt” + Work file filename, Default: 'worktodo.txt' -r RESULTS_FILE, --results-file=RESULTS_FILE - Results file filename, Default: “results.json.txt” for - mfaktc/mfakto or “results.txt” otherwise + Results file filename, Default: 'results.json.txt' for + mfaktc/mfakto or 'results.txt' otherwise -L LOGFILE, --logfile=LOGFILE - Log file filename, Default: “primenet.log” + Log file filename, Default: 'primenet.log' -l LOCALFILE, --local-file=LOCALFILE Local configuration file filename, Default: - “local.ini” + 'local.ini' --archive-proofs=ARCHIVE_DIR Directory to archive PRP proof files after upload, Default: none @@ -98,11 +99,6 @@ Options: GIMPS/PrimeNet User ID. Create a GIMPS/PrimeNet account: https://www.mersenne.org/update/. If you do not want a PrimeNet account, you can use ANONYMOUS. - -p PASSWORD, --password=PASSWORD - Optional GIMPS/PrimeNet Password. Deprecated and not - recommended. Only provide if you want to do manual - testing and not report the progress. This was the - default behavior for old versions of this script. -T WORK_PREFERENCE, --worktype=WORK_PREFERENCE Type of work, Default: 150. Supported work preferences: 2 (Trial factoring), 4 (P-1 factoring), @@ -127,26 +123,28 @@ Options: PRP proof certification work limit in percentage of CPU or GPU time, Default: 10%. Requires the --cert- work option. - --min-exp=MIN_EXP Minimum exponent to get from PrimeNet (2 - - 999,999,999) - --max-exp=MAX_EXP Maximum exponent to get from PrimeNet (2 - - 999,999,999) + --min-exp=MIN_EXP Minimum exponent to get from PrimeNet or TF1G (2 - + 9,999,999,999). TF1G assignments are supported by + setting this flag to 1,000,000,000 or above. + --max-exp=MAX_EXP Maximum exponent to get from PrimeNet or TF1G (2 - + 9,999,999,999) + --bit-min=BIT_MIN Minimum bit level of TF1G assignments to fetch + --bit-max=BIT_MAX Maximum bit level of TF1G assignments to fetch + -m, --mlucas Get assignments for Mlucas. -g, --gpuowl, --prpll - Get assignments for GpuOwl or PRPLL instead of Mlucas. - PRPLL is not yet fully supported. - --cudalucas Get assignments for CUDALucas instead of Mlucas. - --mfaktc Get assignments for mfaktc instead of Mlucas. - --mfakto Get assignments for mfakto instead of Mlucas. + Get assignments for GpuOwl or PRPLL. PRPLL is not yet + fully supported. + --cudalucas Get assignments for CUDALucas. + --mfaktc Get assignments for mfaktc. + --mfakto Get assignments for mfakto. --num-workers=NUM_WORKERS Number of workers (CPU Cores/GPUs), Default: 1 -c CPU, --cpu-num=CPU CPU core or GPU number to get assignments for, Default: 0. Deprecated in favor of the --dir option. -n NUM_CACHE, --num-cache=NUM_CACHE - Number of assignments to cache, Default: 0 - (automatically incremented by 1 when doing manual - testing). Deprecated in favor of the --days-work - option. + Number of assignments to cache, Default: 0. Deprecated + in favor of the --days-work option. -W DAYS_OF_WORK, --days-work=DAYS_OF_WORK Days of work to queue ((0-180] days), Default: 1 day for mfaktc/mfakto or 3 days otherwise. Increases @@ -180,9 +178,8 @@ Options: notification options below. --checkin=HOURS_BETWEEN_CHECKINS Hours to wait between sending assignment progress and - expected completion dates (1-168 hours), Default: 6 - hours. Requires that the instance is registered with - PrimeNet. + expected completion dates (1-168 hours), Default: 1 + hours. -t TIMEOUT, --timeout=TIMEOUT Seconds to wait between updates, Default: 3600 seconds (1 hour). Users with slower internet may want to set a @@ -192,17 +189,19 @@ Options: dates for all assignments and exit. --upload-proofs Report assignment results, upload all PRP proofs and exit. Requires PrimeNet User ID. - --recover-all Recover all assignments and exit. This will overwrite - any existing work files. Requires that the instance is - registered with PrimeNet. + --recover Report assignment results, recover all assignments and + exit. This will overwrite any existing work files. + --recover-all The same as --recover, except for PrimeNet it will + additionally recover expired assignments and for + mersenne.ca it will recover all assignments for all + systems/workers to the first worker. This will + overwrite any existing work files. --register-exponents Prompt for all parameters needed to register one or more specific exponents and exit. --unreserve=EXPONENT Unreserve the exponent and exit. Use this only if you are sure you will not be finishing this exponent. - Requires that the instance is registered with - PrimeNet. - --unreserve-all Unreserve all assignments and exit. Requires that the - instance is registered with PrimeNet. + --unreserve-all Report assignment results, unreserve all assignments + and exit. --no-more-work Prevent this program from getting new assignments and exit. --resume-work Resume getting new assignments after having previously @@ -228,8 +227,7 @@ Options: CPU features, Default: '' --frequency=CPU_SPEED CPU frequency/speed (MHz), Default: 1000 MHz - -m MEMORY, --memory=MEMORY - Total physical memory (RAM) (MiB), Default: 1024 MiB + --memory=MEMORY Total physical memory (RAM) (MiB), Default: 1024 MiB --max-memory=DAY_NIGHT_MEMORY Configured day/night P-1 stage 2 memory (MiB), Default: 921 MiB (90% of physical memory). Required @@ -364,3 +362,5 @@ Thanks to [Daniel Connelly](https://github.com/Danc2050) for updating the PrimeN Thanks to Ernst W. Mayer for helping test and for providing feedback on the Mlucas install script. Thanks to Isaac Terrell for providing the needed PRP proof files to test the proof file uploading feature. + +Thanks to [Tyler Busby](https://github.com/brubsby) for updating the PrimeNet program to support mfaktc/mfakto and getting assignments and reporting results to mersenne.ca for exponents above the PrimeNet limit of 1G. diff --git a/cudalucas.sh b/cudalucas.sh index 96afaa6..a6323d0 100644 --- a/cudalucas.sh +++ b/cudalucas.sh @@ -154,7 +154,7 @@ if command -v nvidia-smi >/dev/null && nvidia-smi >/dev/null; then mapfile -t TOTAL_GPU_MEM < <(nvidia-smi --query-gpu=memory.total --format=csv,noheader,nounits | grep -iv 'not supported') if [[ -n $TOTAL_GPU_MEM ]]; then total=${TOTAL_GPU_MEM[DEVICE]} - ARGS+=(-m "$total" --max-memory="$total") + ARGS+=(--memory="$total" --max-memory="$total") fi fi python3 -OO primenet.py -t 0 -T "$TYPE" -u "$USERID" --cudalucas "cudalucas.out" -H "$COMPUTER" "${ARGS[@]}" diff --git a/cudalucas2.sh b/cudalucas2.sh index 1cf6d6f..e96b02e 100644 --- a/cudalucas2.sh +++ b/cudalucas2.sh @@ -167,7 +167,7 @@ if command -v nvidia-smi >/dev/null && nvidia-smi >/dev/null; then mapfile -t TOTAL_GPU_MEM < <(nvidia-smi --query-gpu=memory.total --format=csv,noheader,nounits | grep -iv 'not supported') if [[ -n $TOTAL_GPU_MEM ]]; then total=${TOTAL_GPU_MEM[DEVICE]} - ARGS+=(-m "$total" --max-memory="$total") + ARGS+=(--memory="$total" --max-memory="$total") fi fi python3 -OO primenet.py -t 0 -T "$TYPE" -u "$USERID" -i "worktodo$N.txt" -r "results$N.txt" -L "primenet$N.log" -l "local$N.ini" --cudalucas "cudalucas$N.out" -H "$COMPUTER" "${ARGS[@]}" diff --git a/gpuowl.sh b/gpuowl.sh index ce2090a..d2000b8 100644 --- a/gpuowl.sh +++ b/gpuowl.sh @@ -223,7 +223,7 @@ if command -v clinfo >/dev/null; then mapfile -t TOTAL_GPU_MEM < <(echo "$clinfo" | sed -n 's/.*CL_DEVICE_GLOBAL_MEM_SIZE *//p') maxAlloc=$((TOTAL_GPU_MEM[DEVICE] >> 20)) - ARGS+=(-m "$maxAlloc" --max-memory="$(echo "$maxAlloc" | awk '{ printf "%d", $1 * 0.9 }')") + ARGS+=(--memory="$maxAlloc" --max-memory="$(echo "$maxAlloc" | awk '{ printf "%d", $1 * 0.9 }')") elif command -v nvidia-smi >/dev/null && nvidia-smi >/dev/null; then mapfile -t GPU < <(nvidia-smi --query-gpu=gpu_name --format=csv,noheader) ARGS+=(--cpu-model="${GPU[DEVICE]}") @@ -236,7 +236,7 @@ elif command -v nvidia-smi >/dev/null && nvidia-smi >/dev/null; then mapfile -t TOTAL_GPU_MEM < <(nvidia-smi --query-gpu=memory.total --format=csv,noheader,nounits | grep -iv 'not supported') if [[ -n $TOTAL_GPU_MEM ]]; then maxAlloc=${TOTAL_GPU_MEM[DEVICE]} - ARGS+=(-m "$maxAlloc" --max-memory="$(echo "$maxAlloc" | awk '{ printf "%d", $1 * 0.9 }')") + ARGS+=(--memory="$maxAlloc" --max-memory="$(echo "$maxAlloc" | awk '{ printf "%d", $1 * 0.9 }')") fi fi python3 -OO primenet.py -t 0 -T "$TYPE" -u "$USERID" -i 'worktodo.ini' -r 'results.ini' -g -H "$COMPUTER" "${ARGS[@]}" diff --git a/gpuowl2.sh b/gpuowl2.sh index d439eba..761e7c1 100644 --- a/gpuowl2.sh +++ b/gpuowl2.sh @@ -236,7 +236,7 @@ if command -v clinfo >/dev/null; then mapfile -t TOTAL_GPU_MEM < <(echo "$clinfo" | sed -n 's/.*CL_DEVICE_GLOBAL_MEM_SIZE *//p') maxAlloc=$((TOTAL_GPU_MEM[DEVICE] >> 20)) - ARGS+=(-m "$maxAlloc" --max-memory="$(echo "$maxAlloc" | awk '{ printf "%d", $1 * 0.9 }')") + ARGS+=(--memory="$maxAlloc" --max-memory="$(echo "$maxAlloc" | awk '{ printf "%d", $1 * 0.9 }')") elif command -v nvidia-smi >/dev/null && nvidia-smi >/dev/null; then mapfile -t GPU < <(nvidia-smi --query-gpu=gpu_name --format=csv,noheader) ARGS+=(--cpu-model="${GPU[DEVICE]}") @@ -249,7 +249,7 @@ elif command -v nvidia-smi >/dev/null && nvidia-smi >/dev/null; then mapfile -t TOTAL_GPU_MEM < <(nvidia-smi --query-gpu=memory.total --format=csv,noheader,nounits | grep -iv 'not supported') if [[ -n $TOTAL_GPU_MEM ]]; then maxAlloc=${TOTAL_GPU_MEM[DEVICE]} - ARGS+=(-m "$maxAlloc" --max-memory="$(echo "$maxAlloc" | awk '{ printf "%d", $1 * 0.9 }')") + ARGS+=(--memory="$maxAlloc" --max-memory="$(echo "$maxAlloc" | awk '{ printf "%d", $1 * 0.9 }')") fi fi python3 -OO ../primenet.py -t 0 -T "$TYPE" -u "$USERID" -i 'worktodo.ini' -r 'results.ini' -g -H "$COMPUTER" "${ARGS[@]}" diff --git a/mlucas.sh b/mlucas.sh index d6b5263..6c3f221 100644 --- a/mlucas.sh +++ b/mlucas.sh @@ -497,7 +497,7 @@ for j in "${!threads[@]}"; do done echo -e "\nRegistering computer with PrimeNet\n" total=$((TOTAL_PHYSICAL_MEM >> 10)) -python3 -OO ../primenet.py -t 0 -T "$TYPE" -u "$USERID" --num-workers ${#RUNS[*]} -H "$COMPUTER" --cpu-model="${CPU[0]}" --frequency="$(if [[ -n $CPU_FREQ ]]; then printf "%.0f" "${CPU_FREQ/./$decimal_point}"; else echo "1000"; fi)" -m $total --max-memory="$(echo $total | awk '{ printf "%d", $1 * 0.9 }')" --cores="$CPU_CORES" --hyperthreads="$HP" --l1=$((CPU_CACHE_SIZES[1] ? CPU_CACHE_SIZES[1] >> 10 : 8)) --l2=$((CPU_CACHE_SIZES[2] ? CPU_CACHE_SIZES[2] >> 10 : 512)) --l3=$((CPU_CACHE_SIZES[3] >> 10)) +python3 -OO ../primenet.py -t 0 -T "$TYPE" -u "$USERID" --num-workers ${#RUNS[*]} -m -H "$COMPUTER" --cpu-model="${CPU[0]}" --frequency="$(if [[ -n $CPU_FREQ ]]; then printf "%.0f" "${CPU_FREQ/./$decimal_point}"; else echo "1000"; fi)" --memory=$total --max-memory="$(echo $total | awk '{ printf "%d", $1 * 0.9 }')" --cores="$CPU_CORES" --hyperthreads="$HP" --l1=$((CPU_CACHE_SIZES[1] ? CPU_CACHE_SIZES[1] >> 10 : 8)) --l2=$((CPU_CACHE_SIZES[2] ? CPU_CACHE_SIZES[2] >> 10 : 512)) --l3=$((CPU_CACHE_SIZES[3] >> 10)) maxalloc=$(echo ${#RUNS[*]} | awk '{ printf "%g", 90 / $1 }') args=() for i in "${!RUNS[@]}"; do diff --git a/mprime-python-port/mprime.py b/mprime-python-port/mprime.py index 4144ac1..ccc9123 100644 --- a/mprime-python-port/mprime.py +++ b/mprime-python-port/mprime.py @@ -5,6 +5,13 @@ # ./mprime.py "$USER" "$HOSTNAME" 150 10 # ./mprime.py ANONYMOUS +# /// script +# requires-python = ">=3.5" +# dependencies = [ +# "requests", +# ] +# /// + import os import platform import re # regular expression matching diff --git a/primenet.py b/primenet.py index 90d1283..bf85602 100644 --- a/primenet.py +++ b/primenet.py @@ -1,5 +1,13 @@ #!/usr/bin/env python3 # -*- coding: utf-8 -*- + +# /// script +# requires-python = ">=2.6" +# dependencies = [ +# "requests", +# ] +# /// + """Automatic assignment handler for Mlucas, GpuOwl/PRPLL, CUDALucas, CUDAPm1, mfaktc and mfakto. [*] Python can be downloaded from https://www.python.org/downloads/ @@ -128,15 +136,6 @@ def median_low(data): return sorts[(length - 1) // 2] -try: - # Python 3.8+ - from math import isqrt -except ImportError: - - def isqrt(x): - return int(math.sqrt(x)) - - try: # Python 3.3+ from math import log2 @@ -155,39 +154,6 @@ def expm1(x): return math.exp(x) - 1 -try: - # Python 3.3+ - from shutil import disk_usage -except ImportError: - # Adapted from: https://code.activestate.com/recipes/577972-disk-usage/ - from collections import namedtuple - - _ntuple_diskusage = namedtuple("usage", "total used free") - - if hasattr(os, "statvfs"): # POSIX - - def disk_usage(path): - st = os.statvfs(path) - free = st.f_bavail * st.f_frsize - total = st.f_blocks * st.f_frsize - used = (st.f_blocks - st.f_bfree) * st.f_frsize - return _ntuple_diskusage(total, used, free) - - elif os.name == "nt": # Windows - - def disk_usage(path): - _, total, free = ctypes.c_ulonglong(), ctypes.c_ulonglong(), ctypes.c_ulonglong() - fun = ( - ctypes.windll.kernel32.GetDiskFreeSpaceExW - if sys.version_info >= (3,) or isinstance(path, unicode) - else ctypes.windll.kernel32.GetDiskFreeSpaceExA - ) - if not fun(path, ctypes.byref(_), ctypes.byref(total), ctypes.byref(free)): - raise ctypes.WinError() - used = total.value - free.value - return _ntuple_diskusage(total.value, used, free.value) - - if sys.platform == "win32": # Windows from ctypes import wintypes @@ -394,6 +360,77 @@ class nvmlMemory_t(ctypes.Structure): _fields_ = (("total", ctypes.c_ulonglong), ("free", ctypes.c_ulonglong), ("used", ctypes.c_ulonglong)) +try: + # Python 3.3+ + from shutil import disk_usage +except ImportError: + # Adapted from: https://code.activestate.com/recipes/577972-disk-usage/ + from collections import namedtuple + + _ntuple_diskusage = namedtuple("usage", "total used free") + + if hasattr(os, "statvfs"): # POSIX + + def disk_usage(path): + st = os.statvfs(path) + free = st.f_bavail * st.f_frsize + total = st.f_blocks * st.f_frsize + used = (st.f_blocks - st.f_bfree) * st.f_frsize + return _ntuple_diskusage(total, used, free) + + elif os.name == "nt": # Windows + ctypes.windll.kernel32.GetDiskFreeSpaceExW.argtypes = ( + wintypes.LPCWSTR, + ctypes.POINTER(ctypes.c_ulonglong), + ctypes.POINTER(ctypes.c_ulonglong), + ctypes.POINTER(ctypes.c_ulonglong), + ) + ctypes.windll.kernel32.GetDiskFreeSpaceExW.restype = wintypes.BOOL + + ctypes.windll.kernel32.GetDiskFreeSpaceExA.argtypes = ( + wintypes.LPCSTR, + ctypes.POINTER(ctypes.c_ulonglong), + ctypes.POINTER(ctypes.c_ulonglong), + ctypes.POINTER(ctypes.c_ulonglong), + ) + ctypes.windll.kernel32.GetDiskFreeSpaceExA.restype = wintypes.BOOL + + def disk_usage(path): + _, total, free = ctypes.c_ulonglong(), ctypes.c_ulonglong(), ctypes.c_ulonglong() + fun = ( + ctypes.windll.kernel32.GetDiskFreeSpaceExW + if sys.version_info >= (3,) or isinstance(path, unicode) + else ctypes.windll.kernel32.GetDiskFreeSpaceExA + ) + if not fun(path, ctypes.byref(_), ctypes.byref(total), ctypes.byref(free)): + raise ctypes.WinError() + used = total.value - free.value + return _ntuple_diskusage(total.value, used, free.value) + + +try: + # Python 3.3+ + from os import replace +except ImportError: + if os.name == "nt": # Windows + ctypes.windll.kernel32.MoveFileExW.argtypes = (wintypes.LPCWSTR, wintypes.LPCWSTR, wintypes.DWORD) + ctypes.windll.kernel32.MoveFileExW.restype = wintypes.BOOL + + ctypes.windll.kernel32.MoveFileExA.argtypes = (wintypes.LPCSTR, wintypes.LPCSTR, wintypes.DWORD) + ctypes.windll.kernel32.MoveFileExA.restype = wintypes.BOOL + + def replace(src, dst): + fun = ( + ctypes.windll.kernel32.MoveFileExW + if sys.version_info >= (3,) or isinstance(src, unicode) or isinstance(dst, unicode) + else ctypes.windll.kernel32.MoveFileExA + ) + if not fun(src, dst, 0x1): # MOVEFILE_REPLACE_EXISTING + raise ctypes.WinError() + + else: # POSIX + replace = os.rename + try: # Windows import winsound @@ -493,8 +530,9 @@ def beep(): PORT = 8 if is_64bit else 2 session = requests.Session() # session that maintains our cookies -session.mount('https://', requests.adapters.HTTPAdapter(max_retries=urllib3.util.Retry(3, backoff_factor=1, allowed_methods=None))) -session.mount('http://', requests.adapters.HTTPAdapter(max_retries=urllib3.util.Retry(3, backoff_factor=1, allowed_methods=None))) +# allowed_methods=None +session.mount('https://', requests.adapters.HTTPAdapter(max_retries=urllib3.util.Retry(3, backoff_factor=1))) +session.mount('http://', requests.adapters.HTTPAdapter(max_retries=urllib3.util.Retry(3, backoff_factor=1))) atexit.register(session.close) # Python 2.7.9 and 3.4+ # context = ssl.create_default_context(cafile=certifi.where()) @@ -1665,19 +1703,7 @@ def write_workfile(adir, assignments): with tempfile.NamedTemporaryFile("w", dir=adir, delete=False) as f: # Python 3+: encoding="utf-8" pass write_list_file(f.name, tasks) - # Python 3.3+ - if hasattr(os, "replace"): - os.replace(f.name, workfile) - elif os.name == "nt": # Windows - fun = ( - ctypes.windll.kernel32.MoveFileExW - if sys.version_info >= (3,) or isinstance(f.name, unicode) or isinstance(workfile, unicode) - else ctypes.windll.kernel32.MoveFileExA - ) - if not fun(f.name, workfile, 0x1): # MOVEFILE_REPLACE_EXISTING - raise ctypes.WinError() - else: - os.rename(f.name, workfile) + replace(f.name, workfile) def exponent_to_str(assignment): @@ -1806,7 +1832,7 @@ def send(subject, message, attachments=None, to=None, cc=None, bcc=None, priorit msg.attach(msg_text) for filename, file in attachments: - ctype, encoding = mimetypes.guess_type(filename) + ctype, encoding = mimetypes.guess_type(filename) # guess_file_type(filename) if ctype is None or encoding is not None: ctype = "application/octet-stream" maintype, subtype = ctype.split("/", 1) @@ -2066,7 +2092,7 @@ def get_cpu_model(): def get_cpu_cores_threads(): """Returns the number of CPU cores and threads on the system.""" - # Python 3.4+ + # Python 3.4+, but can be overridden in 3.13+ # threads = os.cpu_count() # threads = multiprocessing.cpu_count() cores = threads = 0 @@ -2393,30 +2419,7 @@ def send_request(guid, args): r = session.get(primenet_v5_burl, params=args, timeout=180) # logging.debug("URL: " + r.url) r.raise_for_status() - result = parse_v5_resp(r.text) - # logging.debug("RESPONSE:\n" + r.text) - if "pnErrorResult" not in result: - logging.error("PnErrorResult value missing. Full response was:\n" + r.text) - return None - if "pnErrorDetail" not in result: - logging.error("PnErrorDetail string missing") - return None - rc = int(result["pnErrorResult"]) - if rc: - resmsg = errors.get(rc, "Unknown error code") - if args["t"] == "ga" and args.get("get_cert_work"): - logging.debug("PrimeNet error {0}: {1}".format(rc, resmsg)) - logging.debug(result["pnErrorDetail"]) - else: - logging.error("PrimeNet error {0}: {1}".format(rc, resmsg)) - logging.error(result["pnErrorDetail"]) - elif result["pnErrorDetail"] != "SUCCESS": - if result["pnErrorDetail"].count("\n"): - logging.info("PrimeNet success code with additional info:") - logging.info(result["pnErrorDetail"]) - else: - logging.info("PrimeNet success code with additional info: {0}".format(result["pnErrorDetail"])) - + text = r.text except Timeout as e: logging.exception(e, exc_info=options.debug) return None @@ -2426,6 +2429,31 @@ def send_request(guid, args): except ConnectionError as e: logging.exception("ERROR connecting to server for request: {0}".format(e), exc_info=options.debug) return None + + result = parse_v5_resp(text) + # logging.debug("RESPONSE:\n" + text) + if "pnErrorResult" not in result: + logging.error("PnErrorResult value missing. Full response was:\n" + text) + return None + if "pnErrorDetail" not in result: + logging.error("PnErrorDetail string missing") + return None + rc = int(result["pnErrorResult"]) + if rc: + resmsg = errors.get(rc, "Unknown error code") + if args["t"] == "ga" and args.get("get_cert_work"): + logging.debug("PrimeNet error {0}: {1}".format(rc, resmsg)) + logging.debug(result["pnErrorDetail"]) + else: + logging.error("PrimeNet error {0}: {1}".format(rc, resmsg)) + logging.error(result["pnErrorDetail"]) + elif result["pnErrorDetail"] != "SUCCESS": + if result["pnErrorDetail"].count("\n"): + logging.info("PrimeNet success code with additional info:") + logging.info(result["pnErrorDetail"]) + else: + logging.info("PrimeNet success code with additional info: {0}".format(result["pnErrorDetail"])) + return result @@ -2435,12 +2463,12 @@ def get_exponent(n): # r = session.get(primenet_baseurl + "report_exponent_simple/", params=args, timeout=180) r = session.get(mersenne_ca_baseurl + "exponent/{0}/json".format(n), params=args, timeout=180) r.raise_for_status() - json = r.json() - + result = r.json() except RequestException as e: logging.exception(e, exc_info=options.debug) return None - return json + + return result FACTOR_LIMITS = ( @@ -4264,31 +4292,31 @@ def upload_proof(adapter, filename): while True: args = {"UserID": options.user_id, "Exponent": exponent, "FileSize": filesize, "FileMD5": fileHash} r = session.get(primenet_baseurl + "proof_upload/", params=args, timeout=180) - json = r.json() - if "error_status" in json: - if json["error_status"] == 409: + result = r.json() + if "error_status" in result: + if result["error_status"] == 409: adapter.error("Proof {0!r} already uploaded".format(filename)) - adapter.error(str(json)) + adapter.error(str(result)) return True adapter.error("Unexpected error during {0!r} upload".format(filename)) - adapter.error(str(json)) + adapter.error(str(result)) return False r.raise_for_status() - if "URLToUse" not in json: + if "URLToUse" not in result: adapter.error("For proof {0!r}, server response missing URLToUse:".format(filename)) - adapter.error(str(json)) + adapter.error(str(result)) return False - if "need" not in json: + if "need" not in result: adapter.error("For proof {0!r}, server response missing need list:".format(filename)) - adapter.error(str(json)) + adapter.error(str(result)) return False - origUrl = json["URLToUse"] + origUrl = result["URLToUse"] baseUrl = "https" + origUrl[4:] if origUrl.startswith("http:") else origUrl - pos, end = next((int(a), b) for a, b in json["need"].items()) + pos, end = next((int(a), b) for a, b in result["need"].items()) if pos > end or end >= filesize: adapter.error("For proof {0!r}, need list entry bad:".format(filename)) - adapter.error(str(json)) + adapter.error(str(result)) return False if pos: @@ -4300,28 +4328,28 @@ def upload_proof(adapter, filename): chunk = f.read(size) args = {"FileMD5": fileHash, "DataOffset": pos, "DataSize": len(chunk), "DataMD5": md5(chunk).hexdigest()} response = session.post(baseUrl, params=args, files={"Data": (None, chunk)}, timeout=180) - json = response.json() - if "error_status" in json: + result = response.json() + if "error_status" in result: adapter.error("Unexpected error during {0!r} upload".format(filename)) - adapter.error(str(json)) + adapter.error(str(result)) return False response.raise_for_status() - if "FileUploaded" in json: + if "FileUploaded" in result: adapter.info("Proof file {0!r} successfully uploaded".format(filename)) return True - if "need" not in json: + if "need" not in result: adapter.error("For proof {0!r}, no entries in need list:".format(filename)) - adapter.error(str(json)) + adapter.error(str(result)) return False - start, end = next((int(a), b) for a, b in json["need"].items()) + start, end = next((int(a), b) for a, b in result["need"].items()) if start <= pos: adapter.error("For proof {0!r}, sending data did not advance need list:".format(filename)) - adapter.error(str(json)) + adapter.error(str(result)) return False pos = start if pos > end or end >= filesize: adapter.error("For proof {0!r}, need list entry bad:".format(filename)) - adapter.error(str(json)) + adapter.error(str(result)) return False except RequestException as e: logging.exception(e, exc_info=options.debug) @@ -4578,8 +4606,8 @@ def program_options(send=False, start=-1, retry_count=0): return program_options(send, tnum, retry_count + 1) if "w" in result: w = int(result["w"]) - aw = next(key for key, value in convert_dict.items() if value == w) if w in convert_dict.values() else w - if aw not in supported: + aw = next(key for key, value in CONVERT_DICT.items() if value == w) if w in CONVERT_DICT.values() else w + if aw not in SUPPORTED: logging.error("Unsupported worktype = {0} for {1}".format(aw, PROGRAM["name"])) sys.exit(1) if tnum < 0: @@ -4840,7 +4868,7 @@ def update_assignment(adapter, cpu_num, assignment, task): bounds = ("MIN", "MID", "MAX") changed = False if assignment.work_type == PRIMENET.WORK_TYPE_PRP and ( - options.convert_prp_to_ll or (not assignment.prp_dblchk and int(options.work_preference[cpu_num]) in convert_dict) + options.convert_prp_to_ll or (not assignment.prp_dblchk and int(options.work_preference[cpu_num]) in CONVERT_DICT) ): adapter.info("Converting from PRP to LL") assignment.work_type = PRIMENET.WORK_TYPE_DBLCHK if assignment.prp_dblchk else PRIMENET.WORK_TYPE_FIRST_LL @@ -4923,7 +4951,9 @@ def update_assignment(adapter, cpu_num, assignment, task): return assignment, task -def get_assignment(adapter, cpu_num, assignment_num=None, get_cert_work=None, min_exp=None, max_exp=None, recover_all=False, retry_count=0): +def get_assignment( + adapter, cpu_num, assignment_num=None, get_cert_work=None, min_exp=None, max_exp=None, recover_all=False, retry_count=0 +): """Get a new assignment from the PrimeNet server.""" guid = get_guid(config) args = primenet_v5_bargs.copy() @@ -6043,10 +6073,11 @@ def parse_result(adapter, adir, resultsfile, sendline): json={"value1": user_name, "value2": buf, "value3": message}, timeout=30, ) + text = r.text except RequestException as e: logging.exception("Backup notification failed: {0}".format(e), exc_info=options.debug) else: - adapter.debug("Backup notification: {0}".format(r.text)) + adapter.debug("Backup notification: {0}".format(text)) if result_type == PRIMENET.AR_LL_PRIME: subject = "‼️ New Mersenne Prime!!! {0} is prime!".format(string_rep) temp = "Mersenne" @@ -6272,7 +6303,7 @@ def is_pyinstaller(): parser = optparse.OptionParser( usage="%prog [options]\nUse -h/--help to see all options\nUse --setup to configure this instance of the program", version="%prog " + VERSION, - description="This program will automatically get and register assignments, report assignment progress and results, upload proof files to and download certification starting values from PrimeNet for the Mlucas, GpuOwl/PRPLL, CUDALucas, mfaktc and mfakto GIMPS programs. It can get assignments and report results to mersenne.ca for exponents above the PrimeNet limit of 1G. It also saves its configuration to a “local.ini” file by default, so it is only necessary to give most of the arguments once. The first time it is run, it will register the current Mlucas/GpuOwl/PRPLL/CUDALucas/mfaktc/mfakto instance with PrimeNet (see the Registering Options below). Then, it will report assignment results, get assignments and upload any proof files to PrimeNet on the --timeout interval, or only once if --timeout is 0. It will additionally report the progress on the --checkin interval.", + description="This program will automatically get and register assignments, report assignment progress and results, upload proof files to and download certification starting values from PrimeNet for the Mlucas, GpuOwl/PRPLL, CUDALucas, mfaktc and mfakto GIMPS programs. It can get assignments and report results to mersenne.ca for exponents above the PrimeNet limit of 1G. It also saves its configuration to a 'local.ini' file by default, so it is only necessary to give most of the arguments once. The first time it is run, it will register the current Mlucas/GpuOwl/PRPLL/CUDALucas/mfaktc/mfakto instance with PrimeNet (see the Registering Options below). Then, it will report assignment results, get assignments and upload any proof files to PrimeNet on the --timeout interval, or only once if --timeout is 0. It will additionally report the progress on the --checkin interval.", ) # options not saved to local.ini @@ -6300,16 +6331,16 @@ def is_pyinstaller(): ) # all other options are saved to local.ini -parser.add_option("-i", "--work-file", dest="worktodo_file", default="worktodo.txt", help="Work file filename, Default: “%default”") +parser.add_option("-i", "--work-file", dest="worktodo_file", default="worktodo.txt", help="Work file filename, Default: '%default'") parser.add_option( "-r", "--results-file", dest="results_file", - help="Results file filename, Default: “results.json.txt” for mfaktc/mfakto or “results.txt” otherwise", + help="Results file filename, Default: 'results.json.txt' for mfaktc/mfakto or 'results.txt' otherwise", ) -parser.add_option("-L", "--logfile", dest="logfile", default="primenet.log", help="Log file filename, Default: “%default”") +parser.add_option("-L", "--logfile", dest="logfile", default="primenet.log", help="Log file filename, Default: '%default'") parser.add_option( - "-l", "--local-file", dest="localfile", default="local.ini", help="Local configuration file filename, Default: “%default”" + "-l", "--local-file", dest="localfile", default="local.ini", help="Local configuration file filename, Default: '%default'" ) parser.add_option( "--archive-proofs", dest="archive_dir", help="Directory to archive PRP proof files after upload, Default: %default" @@ -6450,7 +6481,7 @@ def is_pyinstaller(): "--checkin", dest="hours_between_checkins", type="int", - default=6, + default=1, help="Hours to wait between sending assignment progress and expected completion dates (1-168 hours), Default: %default hours.", ) parser.add_option( @@ -6712,7 +6743,6 @@ def is_pyinstaller(): # Convert mnemonic-form worktypes to corresponding numeric value, check # worktype value vs supported ones: worktypes = { - "GPUTF": PRIMENET.WP_GPU_FACTOR, "Pfactor": PRIMENET.WP_PFACTOR, "SmallestAvail": PRIMENET.WP_LL_FIRST, "DoubleCheck": PRIMENET.WP_LL_DBLCHK, @@ -6725,7 +6755,7 @@ def is_pyinstaller(): } # {"PRP": 150, "PM1": 4, "LL_DC": 101, "PRP_DC": 151, "PRP_WORLD_RECORD": 152, "PRP_100M": 153, "PRP_P1": 154} # this and the above line of code enables us to use words or numbers on the cmdline -supported = frozenset( +SUPPORTED = frozenset( [PRIMENET.WP_FACTOR, PRIMENET.WP_GPU_FACTOR] if options.mfaktc or options.mfakto else ( @@ -6754,11 +6784,11 @@ def is_pyinstaller(): options.work_preference[i] = work_preference = worktypes[work_preference] if not work_preference.isdigit(): parser.error("Unrecognized work preference = {0}".format(work_preference)) - if int(work_preference) not in supported: + if int(work_preference) not in SUPPORTED: parser.error("Unsupported work preference = {0} for {1}".format(work_preference, PROGRAM["name"])) # Convert first time LL worktypes to PRP -convert_dict = { +CONVERT_DICT = { PRIMENET.WP_LL_FIRST: PRIMENET.WP_PRP_FIRST, PRIMENET.WP_LL_WORLD_RECORD: PRIMENET.WP_PRP_WORLD_RECORD, PRIMENET.WP_LL_100M: PRIMENET.WP_PRP_100M, @@ -6767,8 +6797,8 @@ def is_pyinstaller(): work_preference = [] for awork_preference in options.work_preference: awork_preference = int(awork_preference) - if awork_preference in convert_dict: - awork_preference = convert_dict[awork_preference] + if awork_preference in CONVERT_DICT: + awork_preference = CONVERT_DICT[awork_preference] work_preference.append(awork_preference) if len(options.work_preference) == 1 and options.num_workers > 1: