From f4721aca3d2017cdad53d10829028d67a0bdd362 Mon Sep 17 00:00:00 2001 From: Denis Zhdanov Date: Fri, 19 Jan 2024 12:51:10 +0100 Subject: [PATCH] Sparse/fallocate/uid/gid options for whisper-resize.py (#332) * Sparse/fallocate/uid/gid options for whisper-resize.py * Python 2.7 is not available anymore * Adding Python 3.12 * Taming linter * Excluding contrib from tests --- .github/workflows/tests.yml | 12 ++--- bin/whisper-resize.py | 87 ++++++++++++++++++++++++++++++------- tox.ini | 3 +- 3 files changed, 79 insertions(+), 23 deletions(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 8bd10838..c0fe5740 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -13,9 +13,6 @@ jobs: strategy: matrix: include: - - name: "Python 2.7" - python-version: '2.7' - toxenv: "py27" - name: "Pypy 2.7" python-version: 'pypy-2.7' toxenv: 'pypy' @@ -37,6 +34,9 @@ jobs: - name: "Python 3.11" python-version: '3.11' toxenv: 'py3' + - name: "Python 3.12" + python-version: '3.12' + toxenv: 'py3' steps: - uses: actions/checkout@v3 - name: Set up Python ${{ matrix.python-version }} @@ -54,19 +54,19 @@ jobs: run: | tox - name: Linting - if: ${{ matrix.python-version==3.11 }} + if: ${{ matrix.python-version==3.12 }} env: TOXENV: lint run: | tox - name: Run Codecov - if: ${{ matrix.python-version==3.11 }} + if: ${{ matrix.python-version==3.12 }} env: TOXENV: lint run: | pip install codecov codecov - name: Upload coverage to Codecov - if: ${{ matrix.python-version==3.11 }} + if: ${{ matrix.python-version==3.12 }} uses: codecov/codecov-action@v3 diff --git a/bin/whisper-resize.py b/bin/whisper-resize.py index 62a11004..b06acaf8 100755 --- a/bin/whisper-resize.py +++ b/bin/whisper-resize.py @@ -42,6 +42,18 @@ option_parser.add_option( '--newfile', default=None, action='store', help="Create a new database file without removing the existing one") +option_parser.add_option( + '--sparse', default=True, action='store_true', + help="Create new whisper as sparse file") +option_parser.add_option( + '--fallocate', default=False, action='store_true', + help="Create new whisper and use fallocate (disabling sparse)") +option_parser.add_option( + '--chown-uid', default=-1, + type='int', help="Run chown for specific UID") +option_parser.add_option( + '--chown-gid', default=-1, + type='int', help="Run chown for specific GID") option_parser.add_option( '--nobackup', action='store_true', help='Delete the .bak file after successful execution') @@ -49,6 +61,12 @@ '--aggregate', action='store_true', help='Try to aggregate the values to fit the new archive better.' ' Note that this will make things slower and use more memory.') +option_parser.add_option( + '--quiet', default=False, action='store_true', + help='Print less messages') +option_parser.add_option( + '--silent', default=False, action='store_true', + help='Print no messages') (options, args) = option_parser.parse_args() @@ -63,6 +81,12 @@ option_parser.print_help() sys.exit(1) +if not options.silent: + size = os.stat(path).st_size + blocks = os.stat(path).st_blocks + print('Old file: %s (%d bytes, %d blocks*%d=%d bytes on disk)' + % (path, size, blocks, 512, blocks * 512)) + info = whisper.info(path) new_archives = [whisper.parseRetentionDef(retentionDef) @@ -82,7 +106,8 @@ else: aggregationMethod = options.aggregationMethod -print('Retrieving all data from the archives') +if not options.quiet and not options.silent: + print('Retrieving all data from the archives') for archive in old_archives: fromTime = now - archive['retention'] + archive['secondsPerPoint'] untilTime = now @@ -92,20 +117,33 @@ if options.newfile is None: tmpfile = path + '.tmp' if os.path.exists(tmpfile): - print('Removing previous temporary database file: %s' % tmpfile) + if not options.quiet and not options.silent: + print('Removing previous temporary database file: %s' % tmpfile) os.unlink(tmpfile) newfile = tmpfile else: newfile = options.newfile -print('Creating new whisper database: %s' % newfile) -whisper.create(newfile, new_archives, xFilesFactor=xff, aggregationMethod=aggregationMethod) +if not options.quiet and not options.silent: + print('Creating new whisper database: %s' % newfile) + +try: + whisper.create(newfile, new_archives, xFilesFactor=xff, + aggregationMethod=aggregationMethod, sparse=options.sparse, + useFallocate=options.fallocate) +except whisper.WhisperException as exc: + raise SystemExit('[ERROR] %s' % str(exc)) + size = os.stat(newfile).st_size -print('Created: %s (%d bytes)' % (newfile, size)) +blocks = os.stat(newfile).st_blocks +if not options.quiet and not options.silent: + print('Created: %s (%d bytes, %d blocks*%d=%d bytes on disk)' + % (newfile, size, blocks, 512, blocks * 512)) if options.aggregate: # This is where data will be interpolated (best effort) - print('Migrating data with aggregation...') + if not options.quiet and not options.silent: + print('Migrating data with aggregation...') all_datapoints = [] for archive in sorted(old_archives, key=lambda x: x['secondsPerPoint']): # Loading all datapoints into memory for fast querying @@ -126,8 +164,8 @@ oldtimestamps = list(map(lambda p: p[0], all_datapoints)) oldvalues = list(map(lambda p: p[1], all_datapoints)) - - print("oldtimestamps: %s" % oldtimestamps) + if not options.quiet and not options.silent: + print("oldtimestamps: %s" % oldtimestamps) # Simply cleaning up some used memory del all_datapoints @@ -138,9 +176,11 @@ step = archive['secondsPerPoint'] fromTime = now - archive['retention'] + now % step untilTime = now + now % step + step - print("(%s,%s,%s)" % (fromTime, untilTime, step)) + if not options.quiet and not options.silent: + print("(%s,%s,%s)" % (fromTime, untilTime, step)) timepoints_to_update = range(fromTime, untilTime, step) - print("timepoints_to_update: %s" % timepoints_to_update) + if not options.quiet and not options.silent: + print("timepoints_to_update: %s" % timepoints_to_update) newdatapoints = [] for tinterval in zip(timepoints_to_update[:-1], timepoints_to_update[1:]): # TODO: Setting lo= parameter for 'lefti' based on righti from previous @@ -157,7 +197,8 @@ non_none, newvalues)]) whisper.update_many(newfile, newdatapoints) else: - print('Migrating data without aggregation...') + if not options.quiet and not options.silent: + print('Migrating data without aggregation...') for archive in old_archives: timeinfo, values = archive['data'] datapoints = zip(range(*timeinfo), values) @@ -168,18 +209,34 @@ sys.exit(0) backup = path + '.bak' -print('Renaming old database to: %s' % backup) +if not options.quiet and not options.silent: + print('Renaming old database to: %s' % backup) os.rename(path, backup) try: - print('Renaming new database to: %s' % path) + if not options.quiet and not options.silent: + print('Renaming new database to: %s' % path) os.rename(tmpfile, path) except (OSError): traceback.print_exc() - print('\nOperation failed, restoring backup') + if not options.quiet and not options.silent: + print('\nOperation failed, restoring backup') os.rename(backup, path) sys.exit(1) +if options.chown_uid > 0 and options.chown_gid > 0: + try: + os.chown(path=path, uid=options.chown_uid, gid=options.chown_gid) + except (OSError): + traceback.print_exc() + +if not options.silent: + size = os.stat(path).st_size + blocks = os.stat(path).st_blocks + print('New file: %s (%d bytes, %d blocks*%d=%d bytes on disk)' + % (path, size, blocks, 512, blocks * 512)) + if options.nobackup: - print("Unlinking backup: %s" % backup) + if not options.quiet and not options.silent: + print("Unlinking backup: %s" % backup) os.unlink(backup) diff --git a/tox.ini b/tox.ini index 99c6c47d..88a98938 100644 --- a/tox.ini +++ b/tox.ini @@ -9,6 +9,7 @@ envlist = py39, py310, py311, + py312, pypy, pypy3, lint, @@ -25,8 +26,6 @@ commands = sh -c 'bin/whisper-resize.py --help >/dev/null' sh -c 'bin/whisper-set-aggregation-method.py --help >/dev/null' sh -c 'bin/whisper-update.py --help >/dev/null' - sh -c 'contrib/whisper-auto-update.py --help >/dev/null' - sh -c 'contrib/whisper-auto-resize.py --help >/dev/null' coverage run --branch --include='whisper.py,test_whisper.py, bin/whisper-resize.py' test_whisper.py coverage xml coverage report