From d5cd508ca24abc5848b0aa74419a36f8c1550fca Mon Sep 17 00:00:00 2001 From: chrisspen Date: Wed, 22 Feb 2012 11:55:57 -0500 Subject: [PATCH 01/82] Modified database storage to fallback to file storage. Added management commands to handle bulk load and dump of files to/from the filesystem. --- .gitignore | 3 + database_files/management/__init__.py | 0 .../management/commands/__init__.py | 0 .../commands/database_files_dump.py | 42 +++++++++ .../commands/database_files_load.py | 47 ++++++++++ database_files/manager.py | 4 +- database_files/migrations/0001_initial.py | 34 ++++---- database_files/models.py | 26 +++++- database_files/storage.py | 87 ++++++++++++++----- database_files/tests/fixtures/test_data.json | 9 +- database_files/tests/models.py | 3 +- database_files/tests/settings.py | 12 ++- database_files/tests/tests.py | 56 ++++++++++-- database_files/urls.py | 2 +- database_files/views.py | 14 ++- 15 files changed, 278 insertions(+), 61 deletions(-) create mode 100644 database_files/management/__init__.py create mode 100644 database_files/management/commands/__init__.py create mode 100644 database_files/management/commands/database_files_dump.py create mode 100644 database_files/management/commands/database_files_load.py diff --git a/.gitignore b/.gitignore index 57e683c..6b1a720 100644 --- a/.gitignore +++ b/.gitignore @@ -1,2 +1,5 @@ *.pyc /build +.project +.pydevproject +.settings \ No newline at end of file diff --git a/database_files/management/__init__.py b/database_files/management/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/database_files/management/commands/__init__.py b/database_files/management/commands/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/database_files/management/commands/database_files_dump.py b/database_files/management/commands/database_files_dump.py new file mode 100644 index 0000000..11854ce --- /dev/null +++ b/database_files/management/commands/database_files_dump.py @@ -0,0 +1,42 @@ +import os + +from django.conf import settings +from django.core.files.storage import default_storage +from django.core.management.base import BaseCommand, CommandError +from django.db.models import FileField, ImageField + +from database_files.models import File + +from optparse import make_option + +class Command(BaseCommand): + option_list = BaseCommand.option_list + ( + make_option('-w', '--overwrite', action='store_true', + dest='overwrite', default=False, + help='If given, overwrites any existing files.'), + ) + help = 'Dumps all files in the database referenced by FileFields ' + \ + 'or ImageFields onto the filesystem in the directory specified by ' + \ + 'MEDIA_ROOT.' + + def handle(self, *args, **options): + tmp_debug = settings.DEBUG + settings.DEBUG = False + try: + q = File.objects.all() + total = q.count() + i = 0 + for file in q: + i += 1 + print '%i of %i' % (i, total) + fqfn = os.path.join(settings.MEDIA_ROOT, file.name) + fqfn = os.path.normpath(fqfn) + if os.path.isfile(fqfn) and not options['overwrite']: + continue + dirs,fn = os.path.split(fqfn) + if not os.path.isdir(dirs): + os.makedirs(dirs) + open(fqfn, 'wb').write(file.content) + finally: + settings.DEBUG = tmp_debug + \ No newline at end of file diff --git a/database_files/management/commands/database_files_load.py b/database_files/management/commands/database_files_load.py new file mode 100644 index 0000000..f7edd6d --- /dev/null +++ b/database_files/management/commands/database_files_load.py @@ -0,0 +1,47 @@ +import os + +from django.conf import settings +from django.core.files.storage import default_storage +from django.core.management.base import BaseCommand, CommandError +from django.db.models import FileField, ImageField + +from optparse import make_option + +class Command(BaseCommand): + args = '' + help = 'Loads all files on the filesystem referenced by FileFields ' + \ + 'or ImageFields into the database. This should only need to be ' + \ + 'done once, when initially migrating a legacy system.' + + def handle(self, *args, **options): + tmp_debug = settings.DEBUG + settings.DEBUG = False + try: + broken = 0 # Number of db records referencing missing files. + from django.db.models import get_models + for model in get_models(): + for field in model._meta.fields: + if not isinstance(field, (FileField, ImageField)): + continue + print model.__name__, field.name + # Ignore records with null or empty string values. + q = {'%s__isnull'%field.name:False} + xq = {field.name:''} + for row in model.objects.filter(**q).exclude(**xq): + try: + file = getattr(row, field.name) + if file is None: + continue + if not file.name: + continue + if file.path and not os.path.isfile(file.path): + broken += 1 + continue + file.read() + row.save() + except IOError: + broken += 1 + print '-'*80 + print '%i broken' % (broken,) + finally: + settings.DEBUG = tmp_debug diff --git a/database_files/manager.py b/database_files/manager.py index 3271a4e..feb3ae6 100644 --- a/database_files/manager.py +++ b/database_files/manager.py @@ -3,4 +3,6 @@ class FileManager(models.Manager): def get_from_name(self, name): - return self.get(pk=os.path.splitext(os.path.split(name)[1])[0]) +# print 'name:',name +# return self.get(pk=os.path.splitext(os.path.split(name)[1])[0]) + return self.get(name=name) \ No newline at end of file diff --git a/database_files/migrations/0001_initial.py b/database_files/migrations/0001_initial.py index 7d0ebfc..f1368d6 100644 --- a/database_files/migrations/0001_initial.py +++ b/database_files/migrations/0001_initial.py @@ -1,35 +1,37 @@ - +# encoding: utf-8 +import datetime from south.db import db +from south.v2 import SchemaMigration from django.db import models -from database_files.models import * -class Migration: - +class Migration(SchemaMigration): + def forwards(self, orm): # Adding model 'File' db.create_table('database_files_file', ( - ('id', orm['database_files.File:id']), - ('content', orm['database_files.File:content']), - ('size', orm['database_files.File:size']), + ('id', self.gf('django.db.models.fields.AutoField')(primary_key=True)), + ('name', self.gf('django.db.models.fields.CharField')(unique=True, max_length=255, db_index=True)), + ('size', self.gf('django.db.models.fields.PositiveIntegerField')()), + ('_content', self.gf('django.db.models.fields.TextField')(db_column='content')), )) db.send_create_signal('database_files', ['File']) - - - + + def backwards(self, orm): # Deleting model 'File' db.delete_table('database_files_file') - - - + + models = { 'database_files.file': { - 'content': ('django.db.models.fields.TextField', [], {}), + 'Meta': {'object_name': 'File'}, + '_content': ('django.db.models.fields.TextField', [], {'db_column': "'content'"}), 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), - 'size': ('django.db.models.fields.IntegerField', [], {}) + 'name': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '255', 'db_index': 'True'}), + 'size': ('django.db.models.fields.PositiveIntegerField', [], {}) } } - + complete_apps = ['database_files'] diff --git a/database_files/models.py b/database_files/models.py index 496ae46..e182902 100644 --- a/database_files/models.py +++ b/database_files/models.py @@ -1,9 +1,31 @@ +import base64 + from django.db import models + from database_files.manager import FileManager class File(models.Model): - content = models.TextField() - size = models.IntegerField() objects = FileManager() + + name = models.CharField( + max_length=255, + unique=True, + blank=False, + null=False, + db_index=True) + + size = models.PositiveIntegerField( + blank=False, + null=False) + _content = models.TextField(db_column='content') + + @property + def content(self): + return base64.b64decode(self._content) + + @content.setter + def content(self, v): + self._content = base64.b64encode(v) + \ No newline at end of file diff --git a/database_files/storage.py b/database_files/storage.py index 681d2e3..a7021a7 100644 --- a/database_files/storage.py +++ b/database_files/storage.py @@ -1,57 +1,102 @@ -import base64 -from database_files import models -from django.core import files -from django.core.files.storage import Storage -from django.core.urlresolvers import reverse import os import StringIO -class DatabaseStorage(Storage): +from django.conf import settings +from django.core import files +from django.core.files.storage import FileSystemStorage +from django.core.urlresolvers import reverse + +from database_files import models + +class DatabaseStorage(FileSystemStorage): + def _generate_name(self, name, pk): """ Replaces the filename with the specified pk and removes any dir """ - dir_name, file_name = os.path.split(name) - file_root, file_ext = os.path.splitext(file_name) - return '%s%s' % (pk, file_ext) + #dir_name, file_name = os.path.split(name) + #file_root, file_ext = os.path.splitext(file_name) + #return '%s%s' % (pk, file_name) + return name def _open(self, name, mode='rb'): + """ + Open file with filename `name` from the database. + """ try: + # Load file from database. f = models.File.objects.get_from_name(name) + content = f.content + size = f.size except models.File.DoesNotExist: - return None - fh = StringIO.StringIO(base64.b64decode(f.content)) + # If not yet in the database, check the local file system + # and load it into the database if present. + fqfn = self.path(name) + if os.path.isfile(fqfn): + self._save(name, open(fqfn, mode)) + fh = super(DatabaseStorage, self)._open(name, mode) + content = fh.read() + size = fh.size + else: + # Otherwise we don't know where the file is. + return + # Normalize the content to a new file object. + fh = StringIO.StringIO(content) fh.name = name fh.mode = mode - fh.size = f.size - return files.File(fh) + fh.size = size + o = files.File(fh) + return o def _save(self, name, content): + """ + Save file with filename `name` and given content to the database. + """ + full_path = self.path(name) + try: + size = content.size + except AttributeError: + size = os.path.getsize(full_path) f = models.File.objects.create( - content=base64.b64encode(content.read()), - size=content.size, + content=content.read(), + size=size, + name=name, ) return self._generate_name(name, f.pk) def exists(self, name): """ - We generate a new filename for each file, so it will never already - exist. + Returns true if a file with the given filename exists in the database. + Returns false otherwise. """ - return False + if models.File.objects.filter(name=name).count() > 0: + return True + return super(DatabaseStorage, self).exists(name) def delete(self, name): + """ + Deletes the file with filename `name` from the database and filesystem. + """ + full_path = self.path(name) try: models.File.objects.get_from_name(name).delete() except models.File.DoesNotExist: pass + return super(DatabaseStorage, self).delete(name) def url(self, name): - return reverse('database_file', kwargs={'name': name}) + """ + Returns the web-accessible URL for the file with filename `name`. + """ + return os.path.join(settings.MEDIA_URL, name) + #return reverse('database_file', kwargs={'name': name}) def size(self, name): + """ + Returns the size of the file with filename `name` in bytes. + """ + full_path = self.path(name) try: return models.File.objects.get_from_name(name).size except models.File.DoesNotExist: - return 0 - + return super(DatabaseStorage, self).size(name) diff --git a/database_files/tests/fixtures/test_data.json b/database_files/tests/fixtures/test_data.json index 114fafe..5a33203 100644 --- a/database_files/tests/fixtures/test_data.json +++ b/database_files/tests/fixtures/test_data.json @@ -1,10 +1,11 @@ [ { - "pk": 1, - "model": "database_files.file", + "pk": 1, + "model": "database_files.file", "fields": { - "content": "MTIzNDU2Nzg5MA==", + "name": "1.txt", + "_content": "MTIzNDU2Nzg5MA==", "size": 10 } } -] +] \ No newline at end of file diff --git a/database_files/tests/models.py b/database_files/tests/models.py index 91ec2b1..428fcfd 100644 --- a/database_files/tests/models.py +++ b/database_files/tests/models.py @@ -1,5 +1,4 @@ from django.db import models class Thing(models.Model): - upload = models.FileField(upload_to='not required') - + upload = models.FileField(upload_to='i/special') diff --git a/database_files/tests/settings.py b/database_files/tests/settings.py index e454934..4b1447f 100644 --- a/database_files/tests/settings.py +++ b/database_files/tests/settings.py @@ -1,4 +1,13 @@ -DATABASE_ENGINE = 'sqlite3' +import os, sys +PROJECT_DIR = os.path.dirname(__file__) +DATABASES = { + 'default':{ + 'ENGINE': 'django.db.backends.sqlite3', + # Don't do this. It dramatically slows down the test. +# 'NAME': '/tmp/database_files.db', +# 'TEST_NAME': '/tmp/database_files.db', + } +} ROOT_URLCONF = 'database_files.urls' INSTALLED_APPS = [ 'django.contrib.auth', @@ -9,3 +18,4 @@ 'database_files.tests', ] DEFAULT_FILE_STORAGE = 'database_files.storage.DatabaseStorage' +MEDIA_ROOT = os.path.join(PROJECT_DIR, 'media') \ No newline at end of file diff --git a/database_files/tests/tests.py b/database_files/tests/tests.py index 237bdec..551a7f2 100644 --- a/database_files/tests/tests.py +++ b/database_files/tests/tests.py @@ -1,11 +1,50 @@ +import os +import StringIO + from django.core import files from django.test import TestCase +from django.core.files.storage import default_storage + from database_files.models import File from database_files.tests.models import Thing -import StringIO + +DIR = os.path.abspath(os.path.split(__file__)[0]) class DatabaseFilesTestCase(TestCase): def test_adding_file(self): + + # Create default thing storing reference to file + # in the local media directory. + fqfn = os.path.join(DIR,'media/i/special/test.txt') + open(fqfn,'w').write('hello there') + o = Thing() + o.upload = 'i/special/test.txt' + o.save() + id = o.id + + # Confirm thing was saved. + Thing.objects.update() + q = Thing.objects.all() + self.assertEqual(q.count(), 1) + self.assertEqual(q[0].upload.name, 'i/special/test.txt') + + # Confirm the file only exists on the file system + # and hasn't been loaded into the database. + q = File.objects.all() + self.assertEqual(q.count(), 0) + + # Verify we can read the contents of thing. + o = Thing.objects.get(id=id) + self.assertEqual(o.upload.read(), "hello there") + + # Verify that by attempting to read the file, we've automatically + # loaded it into the database. + File.objects.update() + q = File.objects.all() + self.assertEqual(q.count(), 1) + self.assertEqual(q[0].content, "hello there") + + # Load a dynamically created file outside /media. test_file = files.temp.NamedTemporaryFile( suffix='.txt', dir=files.temp.gettempdir() @@ -15,20 +54,27 @@ def test_adding_file(self): t = Thing.objects.create( upload=files.File(test_file), ) - self.assertEqual(File.objects.count(), 1) + self.assertEqual(File.objects.count(), 2) t = Thing.objects.get(pk=t.pk) self.assertEqual(t.upload.file.size, 10) self.assertEqual(t.upload.file.name[-4:], '.txt') self.assertEqual(t.upload.file.read(), '1234567890') t.upload.delete() - self.assertEqual(File.objects.count(), 0) + self.assertEqual(File.objects.count(), 1) + + # Confirm when delete a file from the database, we also delete it from + # the filesystem. + self.assertEqual(default_storage.exists('i/special/test.txt'), True) + default_storage.delete('i/special/test.txt') + self.assertEqual(default_storage.exists('i/special/test.txt'), False) + self.assertEqual(os.path.isfile(fqfn), False) class DatabaseFilesViewTestCase(TestCase): fixtures = ['test_data.json'] def test_reading_file(self): - response = self.client.get('/1.txt') + self.assertEqual(File.objects.count(), 1) + response = self.client.get('/files/1.txt') self.assertEqual(response.content, '1234567890') self.assertEqual(response['content-type'], 'text/plain') self.assertEqual(unicode(response['content-length']), '10') - diff --git a/database_files/urls.py b/database_files/urls.py index 5f575c9..c256013 100644 --- a/database_files/urls.py +++ b/database_files/urls.py @@ -1,5 +1,5 @@ from django.conf.urls.defaults import * urlpatterns = patterns('', - url(r'^(?P.+)$', 'database_files.views.serve', name='database_file'), + url(r'^files/(?P.+)$', 'database_files.views.serve', name='database_file'), ) diff --git a/database_files/views.py b/database_files/views.py index 2f08960..1b0b977 100644 --- a/database_files/views.py +++ b/database_files/views.py @@ -1,20 +1,18 @@ import base64 +import os + from django.http import Http404, HttpResponse from django.shortcuts import get_object_or_404 from django.views.decorators.cache import cache_control + import mimetypes + from database_files.models import File -import os @cache_control(max_age=86400) def serve(request, name): - pk, file_ext = os.path.splitext(name) - try: - pk = int(pk) - except ValueError: - raise Http404('Filename is not an integer') - f = get_object_or_404(File, pk=pk) + f = get_object_or_404(File, name=name) mimetype = mimetypes.guess_type(name)[0] or 'application/octet-stream' - response = HttpResponse(base64.b64decode(f.content), mimetype=mimetype) + response = HttpResponse(f.content, mimetype=mimetype) response['Content-Length'] = f.size return response From 48bda6557acf8bd741c3c25b88aba3334c1cea3c Mon Sep 17 00:00:00 2001 From: chrisspen Date: Wed, 22 Feb 2012 17:01:23 -0500 Subject: [PATCH 02/82] Added data migration to auto-load existing files into the database. Updated documentation to note differences from parent. --- README.md | 31 +++++++++++++------ .../commands/database_files_load.py | 3 +- database_files/migrations/0001_initial.py | 7 +++-- database_files/migrations/0002_load_files.py | 30 ++++++++++++++++++ run_tests.sh | 5 ++- setup.py | 29 +++++++++-------- 6 files changed, 75 insertions(+), 30 deletions(-) create mode 100644 database_files/migrations/0002_load_files.py diff --git a/README.md b/README.md index 3cad5eb..e5907b5 100644 --- a/README.md +++ b/README.md @@ -1,7 +1,7 @@ django-database-files ===================== -django-database-files is a storage system for Django that stores uploaded files +django-database-files is a storage system for Django that stores uploaded files in the database. WARNING: It is generally a bad idea to serve static files from Django, @@ -16,25 +16,38 @@ Requires: Installation ------------ - $ python setup.py install + $ sudo python setup.py install + + Or via pip with: + + $ sudo pip install https://github.com/chrisspen/django-database-files/zipball/master Usage ----- -In ``settings.py``, add ``database_files`` to your ``INSTALLED_APPS`` and add this line: +In ``settings.py``, add ``database_files`` to your ``INSTALLED_APPS`` and add +this line: DEFAULT_FILE_STORAGE = 'database_files.storage.DatabaseStorage' -Although ``upload_to`` is a required argument on ``FileField``, it is not used for -storing files in the database. Just set it to a dummy value: +Note, the ``upload_to`` parameter is still used to synchronize the files stored +in the database with those on the file system, so new and existing fields +should still have a value that makes sense from your base media directory. - upload = models.FileField(upload_to='not required') +If you're using South, the initial model migrations will scan through all +existing models for ``FileFields`` or ``ImageFields`` and will automatically +load them into the database. -All your ``FileField`` and ``ImageField`` files will now be stored in the -database. +If for any reason you want to re-run this bulk import task, run: + + $ python manage.py database_files_load + +Additionally, if you want to export all files in the database back to the file +system, run: + + $ python manage.py database_files_dump Test suite ---------- $ ./run_tests.sh - diff --git a/database_files/management/commands/database_files_load.py b/database_files/management/commands/database_files_load.py index f7edd6d..6aa5aca 100644 --- a/database_files/management/commands/database_files_load.py +++ b/database_files/management/commands/database_files_load.py @@ -3,7 +3,7 @@ from django.conf import settings from django.core.files.storage import default_storage from django.core.management.base import BaseCommand, CommandError -from django.db.models import FileField, ImageField +from django.db.models import FileField, ImageField, get_models from optparse import make_option @@ -18,7 +18,6 @@ def handle(self, *args, **options): settings.DEBUG = False try: broken = 0 # Number of db records referencing missing files. - from django.db.models import get_models for model in get_models(): for field in model._meta.fields: if not isinstance(field, (FileField, ImageField)): diff --git a/database_files/migrations/0001_initial.py b/database_files/migrations/0001_initial.py index f1368d6..010f933 100644 --- a/database_files/migrations/0001_initial.py +++ b/database_files/migrations/0001_initial.py @@ -1,8 +1,11 @@ # encoding: utf-8 import datetime + +from django.core.management import call_command +from django.db import models + from south.db import db from south.v2 import SchemaMigration -from django.db import models class Migration(SchemaMigration): @@ -17,13 +20,11 @@ def forwards(self, orm): )) db.send_create_signal('database_files', ['File']) - def backwards(self, orm): # Deleting model 'File' db.delete_table('database_files_file') - models = { 'database_files.file': { 'Meta': {'object_name': 'File'}, diff --git a/database_files/migrations/0002_load_files.py b/database_files/migrations/0002_load_files.py new file mode 100644 index 0000000..58b5a1b --- /dev/null +++ b/database_files/migrations/0002_load_files.py @@ -0,0 +1,30 @@ +# encoding: utf-8 +import datetime + +from django.core.management import call_command +from django.db import models + +from south.db import db +from south.v2 import DataMigration + +class Migration(DataMigration): + + def forwards(self, orm): + # Load any files referenced by existing models into the database. + call_command('database_files_load') + + def backwards(self, orm): + import database_files + database_files.models.File.objects.all().delete() + + models = { + 'database_files.file': { + 'Meta': {'object_name': 'File'}, + '_content': ('django.db.models.fields.TextField', [], {'db_column': "'content'"}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'name': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '255', 'db_index': 'True'}), + 'size': ('django.db.models.fields.PositiveIntegerField', [], {}) + } + } + + complete_apps = ['database_files'] diff --git a/run_tests.sh b/run_tests.sh index 36c3d78..aa7d278 100755 --- a/run_tests.sh +++ b/run_tests.sh @@ -1,3 +1,2 @@ -#!/bin/sh -PYTHONPATH=. DJANGO_SETTINGS_MODULE="database_files.tests.settings" django-admin.py test tests - +#!/bin/bash +PYTHONPATH=. DJANGO_SETTINGS_MODULE="database_files.tests.settings" django-admin.py test tests \ No newline at end of file diff --git a/setup.py b/setup.py index 52eda75..6672661 100644 --- a/setup.py +++ b/setup.py @@ -1,22 +1,25 @@ #!/usr/bin/env python # -*- coding: utf-8 -*- from distutils.core import setup - setup( name='django-database-files', version='0.1', - description='A storage system for Django that stores uploaded files in the database.', - author='Ben Firshman', - author_email='ben@firshman.co.uk', - url='http://github.com/bfirsh/django-database-files/', + description='A storage system for Django that stores uploaded files in both the database and file system.', + author='Chris Spencer', + author_email='chrisspen@gmail.com', + url='http://github.com/chrisspen/django-database-files', packages=[ 'database_files', + 'database_files.management', + 'database_files.management.commands', + 'database_files.migrations', ], - classifiers=['Development Status :: 4 - Beta', - 'Framework :: Django', - 'Intended Audience :: Developers', - 'License :: OSI Approved :: BSD License', - 'Operating System :: OS Independent', - 'Programming Language :: Python', - ], -) + classifiers=[ + 'Development Status :: 4 - Beta', + 'Framework :: Django', + 'Intended Audience :: Developers', + 'License :: OSI Approved :: BSD License', + 'Operating System :: OS Independent', + 'Programming Language :: Python', + ], +) \ No newline at end of file From 081c84507ba37e9b7bc20aa70c8d95443d01de5e Mon Sep 17 00:00:00 2001 From: chrisspen Date: Wed, 22 Feb 2012 17:03:31 -0500 Subject: [PATCH 03/82] Fixed typo. --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index e5907b5..b58ee79 100644 --- a/README.md +++ b/README.md @@ -18,7 +18,7 @@ Installation $ sudo python setup.py install - Or via pip with: +Or via pip with: $ sudo pip install https://github.com/chrisspen/django-database-files/zipball/master From 80991a5696645125606ad2c55b37cc8329913c49 Mon Sep 17 00:00:00 2001 From: chrisspen Date: Wed, 22 Feb 2012 19:19:52 -0500 Subject: [PATCH 04/82] Added management command to handle bulk file deletions. --- README.md | 7 ++++ .../commands/database_files_cleanup.py | 40 +++++++++++++++++++ .../commands/database_files_dump.py | 13 +++--- .../commands/database_files_load.py | 2 - database_files/storage.py | 9 ++++- database_files/utils.py | 16 ++++++++ 6 files changed, 75 insertions(+), 12 deletions(-) create mode 100644 database_files/management/commands/database_files_cleanup.py create mode 100644 database_files/utils.py diff --git a/README.md b/README.md index b58ee79..99d6cac 100644 --- a/README.md +++ b/README.md @@ -47,6 +47,13 @@ system, run: $ python manage.py database_files_dump +Note, that when a field referencing a file is cleared, the corresponding file +in the database and on the file system will not be automatically deleted. +To delete all files in the database and file system not referenced by any model +fields, run: + + $ python manage.py database_files_cleanup + Test suite ---------- diff --git a/database_files/management/commands/database_files_cleanup.py b/database_files/management/commands/database_files_cleanup.py new file mode 100644 index 0000000..74c7553 --- /dev/null +++ b/database_files/management/commands/database_files_cleanup.py @@ -0,0 +1,40 @@ +import os + +from django.conf import settings +from django.core.files.storage import default_storage +from django.core.management.base import BaseCommand, CommandError +from django.db.models import FileField, ImageField, get_models + +from database_files.models import File + +class Command(BaseCommand): + args = '' + help = 'Deletes all files in the database that are not referenced by ' + \ + 'any model fields.' + + def handle(self, *args, **options): + tmp_debug = settings.DEBUG + settings.DEBUG = False + names = set() + try: + for model in get_models(): + for field in model._meta.fields: + if not isinstance(field, (FileField, ImageField)): + continue + # Ignore records with null or empty string values. + q = {'%s__isnull'%field.name:False} + xq = {field.name:''} + for row in model.objects.filter(**q).exclude(**xq): + file = getattr(row, field.name) + if file is None: + continue + if not file.name: + continue + names.add(file.name) + # Find all database files with names not in our list. + orphan_files = File.objects.exclude(name__in=names) + for f in orphan_files: + print 'Deleting %s...' % (f.name,) + default_storage.delete(f.name) + finally: + settings.DEBUG = tmp_debug diff --git a/database_files/management/commands/database_files_dump.py b/database_files/management/commands/database_files_dump.py index 11854ce..57b42bb 100644 --- a/database_files/management/commands/database_files_dump.py +++ b/database_files/management/commands/database_files_dump.py @@ -6,6 +6,7 @@ from django.db.models import FileField, ImageField from database_files.models import File +from database_files.utils import write_file from optparse import make_option @@ -29,14 +30,10 @@ def handle(self, *args, **options): for file in q: i += 1 print '%i of %i' % (i, total) - fqfn = os.path.join(settings.MEDIA_ROOT, file.name) - fqfn = os.path.normpath(fqfn) - if os.path.isfile(fqfn) and not options['overwrite']: - continue - dirs,fn = os.path.split(fqfn) - if not os.path.isdir(dirs): - os.makedirs(dirs) - open(fqfn, 'wb').write(file.content) + write_file( + file.name, + file.content.read(), + options['overwrite']) finally: settings.DEBUG = tmp_debug \ No newline at end of file diff --git a/database_files/management/commands/database_files_load.py b/database_files/management/commands/database_files_load.py index 6aa5aca..7f52903 100644 --- a/database_files/management/commands/database_files_load.py +++ b/database_files/management/commands/database_files_load.py @@ -5,8 +5,6 @@ from django.core.management.base import BaseCommand, CommandError from django.db.models import FileField, ImageField, get_models -from optparse import make_option - class Command(BaseCommand): args = '' help = 'Loads all files on the filesystem referenced by FileFields ' + \ diff --git a/database_files/storage.py b/database_files/storage.py index a7021a7..4831095 100644 --- a/database_files/storage.py +++ b/database_files/storage.py @@ -7,6 +7,7 @@ from django.core.urlresolvers import reverse from database_files import models +from database_files.utils import write_file class DatabaseStorage(FileSystemStorage): @@ -57,11 +58,16 @@ def _save(self, name, content): size = content.size except AttributeError: size = os.path.getsize(full_path) + content = content.read() f = models.File.objects.create( - content=content.read(), + content=content, size=size, name=name, ) + # Automatically write the change to the local file system. + if getattr(settings, 'DATABASE_FILES_FS_AUTO_WRITE', True): + write_file(name, content, overwrite=True) + #TODO:add callback to handle custom save behavior? return self._generate_name(name, f.pk) def exists(self, name): @@ -77,7 +83,6 @@ def delete(self, name): """ Deletes the file with filename `name` from the database and filesystem. """ - full_path = self.path(name) try: models.File.objects.get_from_name(name).delete() except models.File.DoesNotExist: diff --git a/database_files/utils.py b/database_files/utils.py new file mode 100644 index 0000000..9ac3c24 --- /dev/null +++ b/database_files/utils.py @@ -0,0 +1,16 @@ +import os + +from django.conf import settings + +def write_file(name, content, overwrite=False): + """ + Writes the given content to the relative filename under the MEDIA_ROOT. + """ + fqfn = os.path.join(settings.MEDIA_ROOT, name) + fqfn = os.path.normpath(fqfn) + if os.path.isfile(fqfn) and not overwrite: + return + dirs,fn = os.path.split(fqfn) + if not os.path.isdir(dirs): + os.makedirs(dirs) + open(fqfn, 'wb').write(content) From 478ff913a5fb7966e122c91d1a623f40351c67b9 Mon Sep 17 00:00:00 2001 From: chrisspen Date: Wed, 29 Feb 2012 11:12:02 -0500 Subject: [PATCH 05/82] Updated version number. --- database_files/__init__.py | 2 ++ setup.py | 3 ++- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/database_files/__init__.py b/database_files/__init__.py index e69de29..fc6f1b8 100644 --- a/database_files/__init__.py +++ b/database_files/__init__.py @@ -0,0 +1,2 @@ +VERSION = (0, 1, 1) +__version__ = '.'.join(map(str, VERSION)) \ No newline at end of file diff --git a/setup.py b/setup.py index 6672661..c0d1664 100644 --- a/setup.py +++ b/setup.py @@ -1,9 +1,10 @@ #!/usr/bin/env python # -*- coding: utf-8 -*- from distutils.core import setup +import database_files setup( name='django-database-files', - version='0.1', + version=database_files.__version__, description='A storage system for Django that stores uploaded files in both the database and file system.', author='Chris Spencer', author_email='chrisspen@gmail.com', From 34a880eb6488c16662891582ca36ef665aede7d0 Mon Sep 17 00:00:00 2001 From: chrisspen Date: Wed, 29 Feb 2012 14:49:07 -0500 Subject: [PATCH 06/82] Fixed bug in dump management command. --- database_files/management/commands/database_files_dump.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/database_files/management/commands/database_files_dump.py b/database_files/management/commands/database_files_dump.py index 57b42bb..35cf624 100644 --- a/database_files/management/commands/database_files_dump.py +++ b/database_files/management/commands/database_files_dump.py @@ -32,7 +32,7 @@ def handle(self, *args, **options): print '%i of %i' % (i, total) write_file( file.name, - file.content.read(), + file.content, options['overwrite']) finally: settings.DEBUG = tmp_debug From ac76488d021bd023d2d4870865766b7fc3c5d3ed Mon Sep 17 00:00:00 2001 From: chrisspen Date: Wed, 29 Feb 2012 14:51:31 -0500 Subject: [PATCH 07/82] Updated version. --- database_files/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/database_files/__init__.py b/database_files/__init__.py index fc6f1b8..2f80e77 100644 --- a/database_files/__init__.py +++ b/database_files/__init__.py @@ -1,2 +1,2 @@ -VERSION = (0, 1, 1) +VERSION = (0, 1, 2) __version__ = '.'.join(map(str, VERSION)) \ No newline at end of file From f32d21b1d4ca2e5b6fcf2ac69288e1002afee970 Mon Sep 17 00:00:00 2001 From: chrisspen Date: Fri, 2 Mar 2012 12:03:05 -0500 Subject: [PATCH 08/82] Added option to load command to allow limiting file load by app model. --- database_files/__init__.py | 2 +- .../commands/database_files_load.py | 29 ++++++++++++++++--- 2 files changed, 26 insertions(+), 5 deletions(-) diff --git a/database_files/__init__.py b/database_files/__init__.py index 2f80e77..094bae7 100644 --- a/database_files/__init__.py +++ b/database_files/__init__.py @@ -1,2 +1,2 @@ -VERSION = (0, 1, 2) +VERSION = (0, 1, 3) __version__ = '.'.join(map(str, VERSION)) \ No newline at end of file diff --git a/database_files/management/commands/database_files_load.py b/database_files/management/commands/database_files_load.py index 7f52903..6139c16 100644 --- a/database_files/management/commands/database_files_load.py +++ b/database_files/management/commands/database_files_load.py @@ -5,19 +5,35 @@ from django.core.management.base import BaseCommand, CommandError from django.db.models import FileField, ImageField, get_models +from optparse import make_option + class Command(BaseCommand): - args = '' + option_list = BaseCommand.option_list + ( + make_option('-m', '--models', + dest='models', default='', + help='A list of models to search for file fields. Default is all.'), + ) help = 'Loads all files on the filesystem referenced by FileFields ' + \ 'or ImageFields into the database. This should only need to be ' + \ 'done once, when initially migrating a legacy system.' def handle(self, *args, **options): + show_files = int(options.get('verbosity', 1)) >= 2 + all_models = [ + _.lower().strip() + for _ in options.get('models', '').split() + if _.strip() + ] tmp_debug = settings.DEBUG settings.DEBUG = False try: broken = 0 # Number of db records referencing missing files. for model in get_models(): - for field in model._meta.fields: + key = "%s.%s" % (model._meta.app_label,model._meta.module_name) + key = key.lower() + if all_models and key not in all_models: + continue + for field in model._meta.fields: if not isinstance(field, (FileField, ImageField)): continue print model.__name__, field.name @@ -31,14 +47,19 @@ def handle(self, *args, **options): continue if not file.name: continue + if show_files: + print "\t",file.name if file.path and not os.path.isfile(file.path): + if show_files: + print "Broken:",file.name broken += 1 continue file.read() row.save() except IOError: broken += 1 - print '-'*80 - print '%i broken' % (broken,) + if show_files: + print '-'*80 + print '%i broken' % (broken,) finally: settings.DEBUG = tmp_debug From a44a0df7b48b8e1066cc289caf628cad6ec0e3a5 Mon Sep 17 00:00:00 2001 From: chris Date: Fri, 6 Apr 2012 15:21:48 -0400 Subject: [PATCH 09/82] Limited debugging output. --- database_files/management/commands/database_files_dump.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/database_files/management/commands/database_files_dump.py b/database_files/management/commands/database_files_dump.py index 35cf624..9e0032a 100644 --- a/database_files/management/commands/database_files_dump.py +++ b/database_files/management/commands/database_files_dump.py @@ -29,7 +29,8 @@ def handle(self, *args, **options): i = 0 for file in q: i += 1 - print '%i of %i' % (i, total) + if not i % 100: + print '%i of %i' % (i, total) write_file( file.name, file.content, From c35eb57e406561e8e0c9d3b85e13d6563b8cf912 Mon Sep 17 00:00:00 2001 From: chris Date: Sun, 15 Apr 2012 12:42:03 -0400 Subject: [PATCH 10/82] Fixed unittest command. --- README.md | 13 ++++++------- database_files/__init__.py | 2 +- database_files/tests/.gitignore | 1 + database_files/tests/tests.py | 5 ++++- run_tests.sh | 2 -- setup.py | 17 ++++++++++++++++- 6 files changed, 28 insertions(+), 12 deletions(-) create mode 100644 database_files/tests/.gitignore delete mode 100755 run_tests.sh diff --git a/README.md b/README.md index 99d6cac..a5754de 100644 --- a/README.md +++ b/README.md @@ -16,11 +16,15 @@ Requires: Installation ------------ - $ sudo python setup.py install + sudo python setup.py install Or via pip with: - $ sudo pip install https://github.com/chrisspen/django-database-files/zipball/master + sudo pip install https://github.com/chrisspen/django-database-files/zipball/master + +You can run unittests with: + + python setup.py test Usage ----- @@ -53,8 +57,3 @@ To delete all files in the database and file system not referenced by any model fields, run: $ python manage.py database_files_cleanup - -Test suite ----------- - - $ ./run_tests.sh diff --git a/database_files/__init__.py b/database_files/__init__.py index 094bae7..d9dfc28 100644 --- a/database_files/__init__.py +++ b/database_files/__init__.py @@ -1,2 +1,2 @@ -VERSION = (0, 1, 3) +VERSION = (0, 1, 4) __version__ = '.'.join(map(str, VERSION)) \ No newline at end of file diff --git a/database_files/tests/.gitignore b/database_files/tests/.gitignore new file mode 100644 index 0000000..9938aa6 --- /dev/null +++ b/database_files/tests/.gitignore @@ -0,0 +1 @@ +/media diff --git a/database_files/tests/tests.py b/database_files/tests/tests.py index 551a7f2..d0531f3 100644 --- a/database_files/tests/tests.py +++ b/database_files/tests/tests.py @@ -15,7 +15,10 @@ def test_adding_file(self): # Create default thing storing reference to file # in the local media directory. - fqfn = os.path.join(DIR,'media/i/special/test.txt') + media_dir = os.path.join(DIR,'media/i/special') + if not os.path.isdir(media_dir): + os.makedirs(media_dir) + fqfn = os.path.join(media_dir,'test.txt') open(fqfn,'w').write('hello there') o = Thing() o.upload = 'i/special/test.txt' diff --git a/run_tests.sh b/run_tests.sh deleted file mode 100755 index aa7d278..0000000 --- a/run_tests.sh +++ /dev/null @@ -1,2 +0,0 @@ -#!/bin/bash -PYTHONPATH=. DJANGO_SETTINGS_MODULE="database_files.tests.settings" django-admin.py test tests \ No newline at end of file diff --git a/setup.py b/setup.py index c0d1664..2236423 100644 --- a/setup.py +++ b/setup.py @@ -1,7 +1,19 @@ #!/usr/bin/env python # -*- coding: utf-8 -*- -from distutils.core import setup +import os +from distutils.core import setup, Command import database_files + +class TestCommand(Command): + description = "Runs unittests." + user_options = [] + def initialize_options(self): + pass + def finalize_options(self): + pass + def run(self): + os.system('django-admin.py test --pythonpath=. --settings=database_files.tests.settings tests') + setup( name='django-database-files', version=database_files.__version__, @@ -23,4 +35,7 @@ 'Operating System :: OS Independent', 'Programming Language :: Python', ], + cmdclass={ + 'test': TestCommand, + }, ) \ No newline at end of file From 8bad5c323db16a1fc1392901532bdec01cf30f88 Mon Sep 17 00:00:00 2001 From: chris Date: Mon, 11 Jun 2012 13:10:52 -0400 Subject: [PATCH 11/82] Added feature to define custom user and group to set ownership with when writing files. --- database_files/utils.py | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/database_files/utils.py b/database_files/utils.py index 9ac3c24..8f106d6 100644 --- a/database_files/utils.py +++ b/database_files/utils.py @@ -1,3 +1,5 @@ +#from grp import getgrnam +#from pwd import getpwnam import os from django.conf import settings @@ -14,3 +16,16 @@ def write_file(name, content, overwrite=False): if not os.path.isdir(dirs): os.makedirs(dirs) open(fqfn, 'wb').write(content) + + # Set ownership and permissions. + uname = getattr(settings, 'DATABASE_FILES_USER', None) + gname = getattr(settings, 'DATABASE_FILES_GROUP', None) + if gname: + gname = ':'+gname + if uname: + os.system('chown -RL %s%s "%s"' % (uname, gname, dirs)) + + # Set permissions. + perms = getattr(settings, 'DATABASE_FILES_PERMS', None) + if perms: + os.system('chmod -R %s "%s"' % (perms, dirs)) \ No newline at end of file From bc6a7e6da1ce912522704c7fb1730a9d68e4aa88 Mon Sep 17 00:00:00 2001 From: chris Date: Mon, 11 Jun 2012 15:11:28 -0400 Subject: [PATCH 12/82] Updated version number. --- database_files/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/database_files/__init__.py b/database_files/__init__.py index d9dfc28..5b6ef1f 100644 --- a/database_files/__init__.py +++ b/database_files/__init__.py @@ -1,2 +1,2 @@ -VERSION = (0, 1, 4) +VERSION = (0, 1, 5) __version__ = '.'.join(map(str, VERSION)) \ No newline at end of file From 63af3a3d7a9727c70233c3d916cac67f51e15904 Mon Sep 17 00:00:00 2001 From: chris Date: Mon, 6 Aug 2012 15:59:47 -0400 Subject: [PATCH 13/82] Improved performance of database_files_dump by only exporting content from the database if there's a difference in file hashes. --- database_files/__init__.py | 2 +- .../commands/database_files_dump.py | 24 ++++++----- ..._datetime__add_field_file__content_hash.py | 42 +++++++++++++++++++ database_files/migrations/0004_set_hash.py | 36 ++++++++++++++++ database_files/migrations/0005_auto.py | 38 +++++++++++++++++ database_files/models.py | 35 +++++++++++++++- database_files/utils.py | 42 ++++++++++++++++++- setup.py | 11 +++++ 8 files changed, 217 insertions(+), 13 deletions(-) create mode 100644 database_files/migrations/0003_auto__add_field_file_created_datetime__add_field_file__content_hash.py create mode 100644 database_files/migrations/0004_set_hash.py create mode 100644 database_files/migrations/0005_auto.py diff --git a/database_files/__init__.py b/database_files/__init__.py index 5b6ef1f..d16a746 100644 --- a/database_files/__init__.py +++ b/database_files/__init__.py @@ -1,2 +1,2 @@ -VERSION = (0, 1, 5) +VERSION = (0, 1, 6) __version__ = '.'.join(map(str, VERSION)) \ No newline at end of file diff --git a/database_files/management/commands/database_files_dump.py b/database_files/management/commands/database_files_dump.py index 9e0032a..2c10a03 100644 --- a/database_files/management/commands/database_files_dump.py +++ b/database_files/management/commands/database_files_dump.py @@ -6,15 +6,15 @@ from django.db.models import FileField, ImageField from database_files.models import File -from database_files.utils import write_file +from database_files.utils import write_file, is_fresh from optparse import make_option class Command(BaseCommand): option_list = BaseCommand.option_list + ( - make_option('-w', '--overwrite', action='store_true', - dest='overwrite', default=False, - help='If given, overwrites any existing files.'), +# make_option('-w', '--overwrite', action='store_true', +# dest='overwrite', default=False, +# help='If given, overwrites any existing files.'), ) help = 'Dumps all files in the database referenced by FileFields ' + \ 'or ImageFields onto the filesystem in the directory specified by ' + \ @@ -24,17 +24,21 @@ def handle(self, *args, **options): tmp_debug = settings.DEBUG settings.DEBUG = False try: - q = File.objects.all() + q = File.objects.all().values_list('id', 'name', '_content_hash') total = q.count() i = 0 - for file in q: + for (file_id, name, content_hash) in q: i += 1 if not i % 100: print '%i of %i' % (i, total) - write_file( - file.name, - file.content, - options['overwrite']) + if not is_fresh(name=name, content_hash=content_hash): + print 'File %i-%s is stale. Writing to local file system...' \ + % (file_id, name) + file = File.objects.get(id=file_id) + write_file( + file.name, + file.content, + overwrite=True) finally: settings.DEBUG = tmp_debug \ No newline at end of file diff --git a/database_files/migrations/0003_auto__add_field_file_created_datetime__add_field_file__content_hash.py b/database_files/migrations/0003_auto__add_field_file_created_datetime__add_field_file__content_hash.py new file mode 100644 index 0000000..1fcf725 --- /dev/null +++ b/database_files/migrations/0003_auto__add_field_file_created_datetime__add_field_file__content_hash.py @@ -0,0 +1,42 @@ +# -*- coding: utf-8 -*- +import datetime +from south.db import db +from south.v2 import SchemaMigration +from django.db import models +from django.utils import timezone + +class Migration(SchemaMigration): + + def forwards(self, orm): + # Adding field 'File.created_datetime' + db.add_column('database_files_file', 'created_datetime', + self.gf('django.db.models.fields.DateTimeField')(default=timezone.now, db_index=True), + keep_default=False) + + # Adding field 'File._content_hash' + db.add_column('database_files_file', '_content_hash', + self.gf('django.db.models.fields.CharField')(max_length=128, null=True, db_column='content_hash', blank=True), + keep_default=False) + + + def backwards(self, orm): + # Deleting field 'File.created_datetime' + db.delete_column('database_files_file', 'created_datetime') + + # Deleting field 'File._content_hash' + db.delete_column('database_files_file', 'content_hash') + + + models = { + 'database_files.file': { + 'Meta': {'object_name': 'File'}, + '_content': ('django.db.models.fields.TextField', [], {'db_column': "'content'"}), + '_content_hash': ('django.db.models.fields.CharField', [], {'max_length': '128', 'null': 'True', 'db_column': "'content_hash'", 'blank': 'True'}), + 'created_datetime': ('django.db.models.fields.DateTimeField', [], {'default': 'timezone.now', 'db_index': 'True'}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'name': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '255', 'db_index': 'True'}), + 'size': ('django.db.models.fields.PositiveIntegerField', [], {}) + } + } + + complete_apps = ['database_files'] \ No newline at end of file diff --git a/database_files/migrations/0004_set_hash.py b/database_files/migrations/0004_set_hash.py new file mode 100644 index 0000000..35431b6 --- /dev/null +++ b/database_files/migrations/0004_set_hash.py @@ -0,0 +1,36 @@ +# -*- coding: utf-8 -*- +import datetime +from south.db import db +from south.v2 import DataMigration +from django.db import models +import base64 + +from database_files import utils + +class Migration(DataMigration): + + def forwards(self, orm): + "Write your forwards methods here." + File = orm['database_files.File'] + q = File.objects.all() + for f in q: + f._content_hash = utils.get_text_hash_0004(base64.b64decode(f._content)) + f.save() + + def backwards(self, orm): + "Write your backwards methods here." + + models = { + 'database_files.file': { + 'Meta': {'object_name': 'File'}, + '_content': ('django.db.models.fields.TextField', [], {'db_column': "'content'"}), + '_content_hash': ('django.db.models.fields.CharField', [], {'max_length': '128', 'null': 'True', 'db_column': "'content_hash'", 'blank': 'True'}), + 'created_datetime': ('django.db.models.fields.DateTimeField', [], {'default': 'datetime.datetime.now', 'db_index': 'True'}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'name': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '255', 'db_index': 'True'}), + 'size': ('django.db.models.fields.PositiveIntegerField', [], {}) + } + } + + complete_apps = ['database_files'] + symmetrical = True diff --git a/database_files/migrations/0005_auto.py b/database_files/migrations/0005_auto.py new file mode 100644 index 0000000..8501fa5 --- /dev/null +++ b/database_files/migrations/0005_auto.py @@ -0,0 +1,38 @@ +# -*- coding: utf-8 -*- +import datetime +from south.db import db +from south.v2 import SchemaMigration +from django.db import models + + +class Migration(SchemaMigration): + + def forwards(self, orm): + # Adding index on 'File', fields ['_content_hash'] + db.create_index('database_files_file', ['content_hash']) + + # Adding index on 'File', fields ['size'] + db.create_index('database_files_file', ['size']) + + + def backwards(self, orm): + # Removing index on 'File', fields ['size'] + db.delete_index('database_files_file', ['size']) + + # Removing index on 'File', fields ['_content_hash'] + db.delete_index('database_files_file', ['content_hash']) + + + models = { + 'database_files.file': { + 'Meta': {'object_name': 'File'}, + '_content': ('django.db.models.fields.TextField', [], {'db_column': "'content'"}), + '_content_hash': ('django.db.models.fields.CharField', [], {'db_index': 'True', 'max_length': '128', 'null': 'True', 'db_column': "'content_hash'", 'blank': 'True'}), + 'created_datetime': ('django.db.models.fields.DateTimeField', [], {'default': 'datetime.datetime.now', 'db_index': 'True'}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'name': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '255', 'db_index': 'True'}), + 'size': ('django.db.models.fields.PositiveIntegerField', [], {'db_index': 'True'}) + } + } + + complete_apps = ['database_files'] \ No newline at end of file diff --git a/database_files/models.py b/database_files/models.py index e182902..6ea11f9 100644 --- a/database_files/models.py +++ b/database_files/models.py @@ -1,7 +1,9 @@ import base64 from django.db import models +from django.utils import timezone +from database_files import utils from database_files.manager import FileManager class File(models.Model): @@ -16,11 +18,36 @@ class File(models.Model): db_index=True) size = models.PositiveIntegerField( + db_index=True, blank=False, null=False) _content = models.TextField(db_column='content') + created_datetime = models.DateTimeField( + db_index=True, + default=timezone.now, + verbose_name="Created datetime") + + _content_hash = models.CharField( + db_column='content_hash', + db_index=True, + max_length=128, + blank=True, null=True) + + def save(self, *args, **kwargs): + + # Check for and clear old content hash. + if self.id: + old = File.objects.get(id=self.id) + if old._content != self._content: + self._content_hash = None + + # Recalculate new content hash. + self.content_hash + + return super(File, self).save(*args, **kwargs) + @property def content(self): return base64.b64decode(self._content) @@ -28,4 +55,10 @@ def content(self): @content.setter def content(self, v): self._content = base64.b64encode(v) - \ No newline at end of file + + @property + def content_hash(self): + if not self._content_hash and self._content: + self._content_hash = utils.get_text_hash(self.content) + return self._content_hash + \ No newline at end of file diff --git a/database_files/utils.py b/database_files/utils.py index 8f106d6..8ee453b 100644 --- a/database_files/utils.py +++ b/database_files/utils.py @@ -1,9 +1,24 @@ #from grp import getgrnam #from pwd import getpwnam import os +import hashlib from django.conf import settings +def is_fresh(name, content_hash): + """ + Returns true if the file exists on the local filesystem and matches the + content in the database. Returns false otherwise. + """ + if not content_hash: + return False + fqfn = os.path.join(settings.MEDIA_ROOT, name) + fqfn = os.path.normpath(fqfn) + if not os.path.isfile(fqfn): + return False + local_content_hash = get_file_hash(fqfn) + return local_content_hash == content_hash + def write_file(name, content, overwrite=False): """ Writes the given content to the relative filename under the MEDIA_ROOT. @@ -28,4 +43,29 @@ def write_file(name, content, overwrite=False): # Set permissions. perms = getattr(settings, 'DATABASE_FILES_PERMS', None) if perms: - os.system('chmod -R %s "%s"' % (perms, dirs)) \ No newline at end of file + os.system('chmod -R %s "%s"' % (perms, dirs)) + +def get_file_hash(fin): + """ + Iteratively builds a file hash without loading the entire file into memory. + """ + if isinstance(fin, basestring): + fin = open(fin) + h = hashlib.sha512() + for text in fin.readlines(): + if not isinstance(text, unicode): + text = unicode(text, encoding='utf-8', errors='replace') + h.update(text.encode('utf-8', 'replace')) + return h.hexdigest() + +def get_text_hash(text): + """ + Returns the hash of the given text. + """ + h = hashlib.sha512() + if not isinstance(text, unicode): + text = unicode(text, encoding='utf-8', errors='replace') + h.update(text.encode('utf-8', 'replace')) + return h.hexdigest() + +get_text_hash_0004 = get_text_hash diff --git a/setup.py b/setup.py index 2236423..131a88a 100644 --- a/setup.py +++ b/setup.py @@ -4,6 +4,16 @@ from distutils.core import setup, Command import database_files +def get_reqs(reqs=[]): + # optparse is included with Python <= 2.7, but has been deprecated in favor + # of argparse. We try to import argparse and if we can't, then we'll add + # it to the requirements + try: + import argparse + except ImportError: + reqs.append("argparse>=1.1") + return reqs + class TestCommand(Command): description = "Runs unittests." user_options = [] @@ -35,6 +45,7 @@ def run(self): 'Operating System :: OS Independent', 'Programming Language :: Python', ], + requires = ["Django (>=1.4)",], cmdclass={ 'test': TestCommand, }, From 249a1df458970eea50f76750677f87a148cc63a0 Mon Sep 17 00:00:00 2001 From: chris Date: Mon, 20 Aug 2012 16:24:36 -0400 Subject: [PATCH 14/82] Added a dryrun option to the cleanup command. Fixed minor bug in content read caused by partially pre-read file object. --- database_files/__init__.py | 2 +- .../commands/database_files_cleanup.py | 20 +++++++++++++++++-- database_files/storage.py | 1 + 3 files changed, 20 insertions(+), 3 deletions(-) diff --git a/database_files/__init__.py b/database_files/__init__.py index d16a746..7631097 100644 --- a/database_files/__init__.py +++ b/database_files/__init__.py @@ -1,2 +1,2 @@ -VERSION = (0, 1, 6) +VERSION = (0, 1, 7) __version__ = '.'.join(map(str, VERSION)) \ No newline at end of file diff --git a/database_files/management/commands/database_files_cleanup.py b/database_files/management/commands/database_files_cleanup.py index 74c7553..ec18fd6 100644 --- a/database_files/management/commands/database_files_cleanup.py +++ b/database_files/management/commands/database_files_cleanup.py @@ -1,4 +1,5 @@ import os +from optparse import make_option from django.conf import settings from django.core.files.storage import default_storage @@ -11,11 +12,20 @@ class Command(BaseCommand): args = '' help = 'Deletes all files in the database that are not referenced by ' + \ 'any model fields.' + option_list = BaseCommand.option_list + ( + make_option('--dryrun', + action='store_true', + dest='dryrun', + default=False, + help='If given, only displays the names of orphaned files ' + \ + 'and does not delete them.'), + ) def handle(self, *args, **options): tmp_debug = settings.DEBUG settings.DEBUG = False names = set() + dryrun = options['dryrun'] try: for model in get_models(): for field in model._meta.fields: @@ -33,8 +43,14 @@ def handle(self, *args, **options): names.add(file.name) # Find all database files with names not in our list. orphan_files = File.objects.exclude(name__in=names) + total_bytes = 0 for f in orphan_files: - print 'Deleting %s...' % (f.name,) - default_storage.delete(f.name) + total_bytes += f.size + if dryrun: + print 'File %s is orphaned.' % (f.name,) + else: + print 'Deleting orphan file %s...' % (f.name,) + default_storage.delete(f.name) + print '%i total bytes in orphan files.' % total_bytes finally: settings.DEBUG = tmp_debug diff --git a/database_files/storage.py b/database_files/storage.py index 4831095..b798f49 100644 --- a/database_files/storage.py +++ b/database_files/storage.py @@ -58,6 +58,7 @@ def _save(self, name, content): size = content.size except AttributeError: size = os.path.getsize(full_path) + content.seek(0) content = content.read() f = models.File.objects.create( content=content, From 0817d2f8f185c8043aa4ab27ab70776ba25bdb58 Mon Sep 17 00:00:00 2001 From: chris Date: Mon, 20 Aug 2012 19:08:40 -0400 Subject: [PATCH 15/82] Added a unittest for hashing functions. --- database_files/tests/tests.py | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/database_files/tests/tests.py b/database_files/tests/tests.py index d0531f3..e92018a 100644 --- a/database_files/tests/tests.py +++ b/database_files/tests/tests.py @@ -7,6 +7,7 @@ from database_files.models import File from database_files.tests.models import Thing +from database_files import utils DIR = os.path.abspath(os.path.split(__file__)[0]) @@ -72,6 +73,18 @@ def test_adding_file(self): self.assertEqual(default_storage.exists('i/special/test.txt'), False) self.assertEqual(os.path.isfile(fqfn), False) + def test_hash(self): + media_dir = os.path.join(DIR,'media/i/special') + if not os.path.isdir(media_dir): + os.makedirs(media_dir) + fqfn = os.path.join(media_dir,'test.txt') + open(fqfn,'wb').write('hello\nmany things are here\nand then this') + f_hash = utils.get_file_hash(fqfn) + t_hash = utils.get_text_hash(open(fqfn, 'rb').read()) + print f_hash + print t_hash + self.assertEqual(f_hash, t_hash) + class DatabaseFilesViewTestCase(TestCase): fixtures = ['test_data.json'] From cad337e36f4f6fedbd944404c722867b1bd164eb Mon Sep 17 00:00:00 2001 From: chris Date: Tue, 21 Aug 2012 14:18:41 -0400 Subject: [PATCH 16/82] Fixing hashing functions to be consistent in Python2.6. Added management command to regenerate hashes. --- database_files/__init__.py | 2 +- .../commands/database_files_rehash.py | 34 +++++++++++ database_files/tests/tests.py | 45 ++++++++++++--- database_files/utils.py | 56 ++++++++++++++++--- setup.py | 1 + 5 files changed, 121 insertions(+), 17 deletions(-) create mode 100644 database_files/management/commands/database_files_rehash.py diff --git a/database_files/__init__.py b/database_files/__init__.py index 7631097..3bf2ad9 100644 --- a/database_files/__init__.py +++ b/database_files/__init__.py @@ -1,2 +1,2 @@ -VERSION = (0, 1, 7) +VERSION = (0, 1, 8) __version__ = '.'.join(map(str, VERSION)) \ No newline at end of file diff --git a/database_files/management/commands/database_files_rehash.py b/database_files/management/commands/database_files_rehash.py new file mode 100644 index 0000000..d12bb63 --- /dev/null +++ b/database_files/management/commands/database_files_rehash.py @@ -0,0 +1,34 @@ +from optparse import make_option + +from django.conf import settings +from django.core.management.base import BaseCommand, CommandError + +from database_files.models import File + +class Command(BaseCommand): + args = '' + help = 'Regenerates hashes for all files.' + option_list = BaseCommand.option_list + ( +# make_option('--dryrun', +# action='store_true', +# dest='dryrun', +# default=False, +# help='If given, only displays the names of orphaned files ' + \ +# 'and does not delete them.'), + ) + + def handle(self, *args, **options): + tmp_debug = settings.DEBUG + settings.DEBUG = False + dryrun = options['dryrun'] + try: + q = File.objects.all() + total = q.count() + i = 1 + for f in q: + print '%i of %i: %s' % (i, total, f.name) + f._content_hash = None + f.save() + i += 1 + finally: + settings.DEBUG = tmp_debug diff --git a/database_files/tests/tests.py b/database_files/tests/tests.py index e92018a..8eee539 100644 --- a/database_files/tests/tests.py +++ b/database_files/tests/tests.py @@ -1,3 +1,4 @@ +# -*- coding: utf-8 -*- import os import StringIO @@ -74,16 +75,44 @@ def test_adding_file(self): self.assertEqual(os.path.isfile(fqfn), False) def test_hash(self): - media_dir = os.path.join(DIR,'media/i/special') + + # Create test file. + media_dir = os.path.join(DIR, 'media/i/special') if not os.path.isdir(media_dir): os.makedirs(media_dir) - fqfn = os.path.join(media_dir,'test.txt') - open(fqfn,'wb').write('hello\nmany things are here\nand then this') - f_hash = utils.get_file_hash(fqfn) - t_hash = utils.get_text_hash(open(fqfn, 'rb').read()) - print f_hash - print t_hash - self.assertEqual(f_hash, t_hash) + image_content = '\x89PNG\r\n\x1a\n\x00\x00\x00\rIHDR\x00\x00\x002\x00\x00\x009\x08\x06\x00\x00\x00t\xf8xr\x00\x00\x00\x01sRGB\x00\xae\xce\x1c\xe9\x00\x00\x00\x06bKGD\x00\xff\x00\xff\x00\xff\xa0\xbd\xa7\x93\x00\x00\x00\tpHYs\x00\x00\x1e\xc2\x00\x00\x1e\xc2\x01n\xd0u>\x00\x00\x00\x07tIME\x07\xdc\x08\x15\r0\x02\xc7\x0f^\x14\x00\x00\x0eNIDATh\xde\xd5\x9aytTU\x9e\xc7?\xf7\xbdWkRd\x0f\x01\x12\x02!\xc8\x120,\r6\xb2#\x10\xa2(\xeeK\xdb:n\xd3\xca\xd0\xb46\x8b\xf6\xc09=\xd3v\xdb\xa7\x91\xb8\xb5L\xab\xe3\xb8\xb5\x8c\xbd\x88\x83\xda-\x84\xc4\xb84\x13\x10F\x01\r\xb2%\x01C \tY\x8a$UIj{\xef\xce\x1f\xaf\xaa\x92\x90\x80@\xc0\xa6\xef9uRU\xa9\xba\xf7\xf7\xbd\xdf\xdf\xfe+\xb8Xk\xc1\x8aD\xe6/K\x03 \x7f%\x17{)\x17kcUU/s%\xa6\xbce[\xf8h6\x9b\xd7B\xfe\x8a\x7fL \xe9)I\t\xd3\'O\x9c-\xe2\x07\xfc\x89\x9c\xd9N6\x17\\Tf.\x1a\x90\xe1\x19\xa9\xa3\xe6\x8d\x19\x84\x96>f\x02#\xe6l\x02`\xf3ZX\xb0\xe2\x1f\x08\xc8\x84\xfb\xfb\xa5\xc6\xc7.\x9c;&\x1d\xabE\x95d\x8c\x9f\xc1\r\xbfx\x95\xf4\\\x85\xc2\x82\x8b\x02F\xb9\x80\xc6M\xcc\xf5\xab\x01p\x0eI\x9f1kl\xd6\xeca)\xfd\xb0)\x8a@\xb3\n\x06\xe4\xdc\xcb\x15\xb7\xad\xe1\xca\x1f:L0\xcb\xb9\xf4V\xd7\x1b\xce\xbec\xfe\xea7\x8bO\x18R\xca\x0f\xcb\xdd\x86kU\x89d\xd9\x16\x83\xe5E\x92%\xefHnyr#\xdf\xbb%\xbd\xf3\xbb\x17\x06\x90\xda\xe7\x1d\xf2W@a\x81\xf9|\xe6OV\xad\x7fz\xc9\xb3?\xbe\xe6\xfbI\xfb\x1b\xda\xe5\xafK\xbe\x11_\xd7y\x91\x08\x01H4\x9b\xc0\x95<\x12W\xf2\x1d$\x0fm\xe6\xc8\xce\xddTl7/\xa2b[\x9f\xc4\x10\xe7\xcd@D\xf8\xa1\xb7*\xd6\x9c\xac\x01\xb9C\x07\xbc\xf8\xdc\xe2E\x0bs\xb2\xd2)>\xe4f\xcb\xc1&^\xdfy\x8c\xa0n`\xe2\x08/)%B\x08:Z\xe1D\xf9\x0e\x02\xde\xd5H\xb6q\xb2\xdaO\xe9\xef\x8d\xef\x96\x91\xc8\xed\r\xbfcX\xde\xeds\xef\xbf\xe1\xca\xb1\x7f~n\xf1\xf5cGe\xa4\xcaV\x7f\x88=5mB\x02\xfb\xeb\xdb\xf0\xfaC\xdd\x81\x880;\x16\xbb ~`:V\xe7\xdd(\xea-\xc4$\xda\xc9\xc8m\xa1b[C\xf4\xb2\xce\x81%q.*\xe4Tm\xb4\xff\xf5\t`J\xf2e\xf7\xdf\xf0\xf0\xe2k\xa7\xdc>m\xf4\x90\xec\xef\rO\x07\x90\x80\x08\xea\x06\xaf}QGY\x8d\x87\xff\xfa\xec\x18\xbe\xa0\xde\x1dH\'3 \x84\x0c\x8b \xf0{\xc1\xd7Z\x83\xbbz\x03u\x07\xd6\xb2m\xfd1~\xf0[p\x1f\xedd\xbf\xcf@\xf2W\x9a1\x00`\xd2\x8f\xee[ro\xfe\x13\x8f\xdd<+e@\xa2K\xd5T5\n\x02\xe0\xbd\xaf\x1b\xa9h\xea\xe0\xc5\xd2\xa3T4\xb4\x99\xef\x8a3\x1c\x13\x05\x84@J0t\x89\xbb\xba\x9e\xe3e\x8fQ\xb2\xee\r\xf3\xfc\xe5\xb0\xf9\xa9>\x00\xe9\n`\xce\xc3\xc9C3\x07\xbc\xf9\xc6\xb2[\x17L\x1f3T\x86\xd5]tj\x0b|\\\xd9\xcc\xd7\'\xdax\xb7\xac\x8e\x92\xfd\x8d\xa0\x9d\x87w7\x81A[3\xd4\xee[G[\xd3c|\xf8|;\xf9\x8f\xc2\xe6\'\xcf\xc3F\xba\x80p,Z5\xfb\x9f\x17^\xf9\xde\x7f?z\xc7\xe4Q\x83S\x91R\n\x11\x96^\x08AP\x97|v\xb4\x95/k\xbd|x\xa8\x91\xc2\xb2z\xb0\x9c\xa7C\x8c\xb0gs\x803~20\x82\xd4\xecR\x8a\x9f\xf5\x90\xff(T\x94\x9e\x03\x90\xaeL,X\xb1d\xdd\x92\x1b^\\~\xe3\x8cA\t\xb1\x0e\xd9\x95E!\x04\xed\x01\x9d\x8f\x0f7SV\xd7\xc6\xa6\x03\rl\xda\xd7\x00\xea\x05\x88\xb3RJ\xacvp\xb8r\xd0\xacS\x89M\xfa\x94\x8f_p\x9f\xce\t\xa8\xbd\xc6\x85\xcda\xe3\x9a\xb1\xf4\xf1\xa2\xb5K~y\xf3\xb4\xb1\xb16\x8b&\xc3L\x10yT7\xfb\xd8t\xc0MM\xab\x9f\xff\xf9\xea\x04\x1f\x957u\xbf\xd5>\x05\x06a\xda\x8cj\x01g|\x06\xd6\x98\xd98b?\xe0\xd3\x97[X\xb0\x1c*\xb6\x7f\x0b\x90\x08\xda\xbc\xe5K\x8b\n\x96<>o|\xb6\x85\x88\xc9\x86=g\xc8\x80]\xc7\xbd|\\\xd9\x8c\xc7\xa7\xf3\xf2\xf6j>\xafj\x06E\\\x18\x10\xdd]\xb5\x00!q\xc6\xf5\xc7\xea\x1cMB\xfa\x07|\xf4\x1f\xbeS\xc1\xa8=\x02]\xc56\x1c\x8bV\xcd}\xe6\xa1\xeb_\xbcm\xfa\xe51\x11\x00B\x08tCR\xe7\t\xf0Ie3\xbbk\xbcT6\xb5\xf3\xfc\xd6*\xaa\xdd\x1d\x17F\x9d\xce\x04HQA\xb3g#uH\xc9,\xc1\xea\xeaf/j7\xbb(,@,X\x9e\xb4h\xfa\xc4\xf7V\xdf6{\x80\xd3\xa6\xc9\x88Q\xd7\xb4\xf8\xf9\xbfc\x1evV{\xa8hj\xa7p\x7f\x03\x1b\xbf:A{P\x07\xe5"\x82\xe8b5Xl\x02_\xcb4B\xa1=\x14\xae=\xc0\x82\x95Q\r\xd2\xba~\x0e\xc0\xe9\x8a{\xfd\xba9S\x86%\xc4\xd8$ \xaaN\xfa\xf8\xaa\xd6K\xbd7H\xc8\x90\x94\x947Rt\xb0\x89\x96\x8eP\xf4;\xdf\xd1\x12\x08Eb\x8b\x15\xd8b~\xcfU\xff\x92I\xe1\xda\x93\x91t\xa9\xbbB\xcfX|\xfd\xac\xebn\xda\xf8\x83\xe99RS\xa4\xe8\x08\x1a\x84\x0cS\xd8\xc3M\xed\xbc\xf5E\r\r\xde\xc0\x85\xb5\x83s\x89/\x8a\nMU\x12_\xb3\xa0\xa5\xfe\x1d6\xad\xb9\xd94\x89\x95(]\xd2h\x8d\xb4\xe1\x05\xce\x84\x14\xfc\xc1\x90hj\x0bR\xef\r\xf0u\xad\x97u[\xabx\xee\xa3\xc34x\xfeN \xa2\x9eP\x824L\xe3w\xc4/`\xfe#SM\x90\x06\x1a"\xac\xdfW?v\x1f\xfd\xb32>)o\x94\r\x1e\x9f\xf0\xeb\x065->\x1a[\xfcf\xf9\xa5*\x7f?\x10\x91e\x18 u\x90R`\xb1\xc4`s=H\xca\x88mly\xcaL1\x18\xb70\x86\xb1W\xbfM\xf2\xd0|\x84\xd2\xa9\xfa\x82\x0b\x17\x17.\xc4\n\x05\xa0\xe9\x1b\x08\xf9\xcd\xd7A\xdfQ<\x8d\x0b)z\xa6\xcc\xa4\xa3\xff\x8818\xe2\'\xa1\xa8\xa6\xd0\x8a\xe8\x8c\t\x97\n\x08\x80\x90\x0f\x8c`\xe7\xe5*\xea`l\xb1S\x99\xf6O(dM\x82\xf8\x81\x13\xd0\xac\xc9\\\xaaKJ\xf3\xe1o\x03#Z\x16HT\x0b(\xcau8\x13\x1d\nC&\xd9\xd1,Wa\x8b\xe5\xbb\xf6\xa7\xe7d\xe8z\x10\xda\x9b;\xf5]J\x11f\xe5*\xac\x0e\xa7B\xbf\xfe6T\xeb,T\x15$\xe2\x92d\x03\x01\xdeFS\xad"\xaa\x1eaE\xb3X\x91r\xa2\x82\xaa%a\xb1\'\xf5\xa1\x82\xbf\xf8l\xf8\xbd\xe0m \xeaa\xbb\x06I\xd5\x02\xd2\x98\xa1\xa0\x07Ga\x8b\x01y\x89\xa9\x95\x0c\x8b\xa3\x07\xc0]\xd5\x1b\x88p\x11\xa6\x82\xa2\xe6*\x18F&\x16; \xc5%i\x17\xee\xea\xae\x06\xde{\x11\xa6hY\nR&\xa3Z/AOe@s\r\x04\xda\xbf5\x97DQ\x924\x90j\x9fc\x85\x94=\xf6\xeefo\xe7\xb2\x7f\xa4f?Y\r\xbe\x96p9"z\xdf7\x9a\xc3k\x0e\xed\xbc-\xdc\x90(\xaa`p\x82\x9d\x1bF\xa72wD"\xd9\xc9N\x12\x9d\x164E\xe1PC\x1b\x7f\xfd\xba\x9e\re\r|s\xb2\x83\x0e\x9fn\xa6:\xa7o\ru2\xe1>\n\x1d\x9e\xce\xf2 |Qi\t1\xf4\xb3\xdbh\x0f\x049\xd1\xd2N0\xa8\x9b\x81\x1bEj\x08\xbc\xe8!P\xb5\xb3\x07\xa1\x1b$\xbbl,\x9d\x96\xc1\xe2\xa9\x19\xa4\xc4\xf4T\xcd\xc9\x83\xe3\x98<8\x8eU\xf3\x86\xf1\xa7\xdd\xb5\xbc\xbc\xa3\x86\xd2\n\xb7Y\x01\x9d\n&\xf2:\xe8\x83\xe6\xe3\x10h\xeb\x04\xe1\x0f\x92\x96\x1a\xc7\x8f\xe6\x8c\xe5\xa6+\x863b`"\xc7\xdc^6|v\x88\x7f\xdb\xb0\x1d\x7f0\x04\xe0S\x199{(\xae\x94\x1b\xd1\xac\xf2\xac\x1c\xb0\x94\x0cJp\xb0\xf1\x9e\\~8q 1V5\xda\x05\xed\xad\x7f`Q\x15\xc6\r\xea\xc7\xa2\x9cT\xfa\xc7\xd9()w#u#|\x93\x116\x14\xf0yLu\n\xf9\xcd\xd7R\x82?\xc8M\xd3G\xf3\xdaCy\xdc:e\x04\xfd\xe3b\xd0\x14\x85\xc4\x18;S/\x1bHK{\x80\xed\xfb\x8e\x81\xe0\x1b\x05E\xad4\rJ\x9c\x15\x08!\x04\x1f\xdc?\x9e)C\xe2\xbb\xb5rO_r\x9b+)\xc6\xc2\xb2\x99\x99\x1c^5\x9d\t\x83\xe3 \x14n\xf3*\nxN\x98\xc9\xa0\x1e\x0exRb\xb7j\xac{(\x8f?,\xbd\x9a\x9c\xf4\xa4\xe8Y]\xcf\xbcf\xc2\x10\x12R\xe2!\xd0Q\xa9\xa0h&\x90\xf0\x06g4\xc2\x90\xc1\xbf\xe7e\x91;\xd0\x15\xddT\x9c\x83!K)\x19\x9c`\xe7\x8b\x9f~\x9f\x9f\xe7g\x13g\x05\xea\x8f@Km\xd4\xee\xac\x9a\xc2\x15\xc3\x07\xf0\xf9\x13w\xb2d\xfe8,\x9a\xda\xe3\xac\xc8\xdf\xf4\x04\x97L\x8dsA\xc8\xbfK\xa3\xbd\xb9\x03G\xdc\x1e\xa41\xee\xcc\xadMHK\xb0\xb3\xfa\xaa\xacs\x06\xd0\xbd\xc3c\xde\xe6/\xf2\x861g\xb0\x9d\x17\x8a\xfcl\xaf\x80@\xc8`tz\x12\xb7^q\x19\x0f\xce\xbd\xbc\xdb\xcd\x9f\xee,\x87U\xc5*$\xa8\x96\x8f5\xbc\x8d>\xe2\x07}H\xd0?\x0e\x8b\xed\xf4v\x122\xb8}|\x1a\xaa"\xce\xa8N\xe7\x02f\xe6\xa8AL\xcaJ\xe5h\xa3\x07\xdd0\x18\x98\x10KB\xac\xfd[U6\xb2\xdauEH#\xd4\x82\xa1\xeeU\xd8\xfaj\x08\xbfg+\x81\xb63g[\x86d\xea\xd0\x84\xf3f\xa370\x00N\x9b\x85\x91\x83\x12\xc9\xc9H\x8e\x828\xcb3\xe4\t\x8f\x1f\x9f\xaf\xa3\x98\xc6\xea\x0e\xd3\xe76T\x96\x91\x90\xbe\x17)\xc7\x9c6\x80II\xb2\xd3\xd2\xe7.hDH\xdd\x90\x1c\xacu\xf3R\xc9Wl\xd8q\x08\xab\xaa\xf2\x93\x05\xe3\xb9gf\x0e\x89\xb1\xf6\xb3aD\x1c\xabk\xa0\xaa\xee\xe4\xfb\xecx\xc5g\x02\xd9\xfa\xda\x11R\x87\x97\xe0J\x19\x8d\xa2\x89^\x99\x11\x82\x86\xb6`\x9f@D\x04\xab\xa8;\xc9K%e\xfc\xae\xf8K\xda=\x1d\xa0\x99\xed\xb5\xe5/\x17\xb3i\xcf\x11\xd6\xdd;\x87\x91\x03\x13M\xe0B\xf4\x10FJI dp\xa4\xaei\x7fp\xfb\xe7\xa5\x00Jt\x90\xd9P\xf1\x9fx\x1a\xda\xa2>\xbcG\x1a \xf80\xdc\xdb\x95R\x9e\xb7:\xfdj\xe3\x0e\xf2\xd7\xbcK\xc1{;i\xef\x08\x98 "\x91\xddi\xa3d\xcf\x11\xae]\xfb.\xa5\x07\x8f\x9b\xc0Os\x96\xd7\x170\xde(\xdaQD\xf5\xdb\x87\xcdNc\xa4\xd7[\xb5\xbb\x81\x01\xa3\xd2I\xcc\x98\x8c\xa2\xc8h\x05\x16Y\x8a`\xd7\xb1VV\xce\x1e\x82US\xce\xca\x18\xbb~\xa6\xf4`\r\xf9\xbf\xd9\xc8\x86\x9d\x87h\xf2t\x80\x94\xb8\x9c6r\x87\xa4\x90\xd9?\x0e_ \x14\x05\xe6\xf6\xfa\xd8\xf2\xd5Q\xa6\x8d\x18Dz\x92\xab\xdb>\xe1\xe7\xf2\x95\xe2]-\xeb\xd7\xac_L\xfb\xc1F\xf2V\x9c\xc2\xda\xe5\xf3,\x0c\x9bYK\xc6\xe5Ih\xb6\x9e\x92\xea\x06\xf3F&\xf3\xee}\xe3p\xf62\xff8\x15\\ \xa4Ss\xb2\x8d_\xbe\xb3\x9dW\x8b\xf6\x80U\x03!\xc8\xee\x1f\xcf3w\xcfb\xe1\x84\xacn\xdf\x7ff\xd3.\xfe\xf5\x8f\xff\x8b?\xa4\x83n02#\x89w\x1e\xb9\x96\xd1\xe1\x80\x18I\x1b\xf7V70\xf6\xfe\'\xd7P\\\xf03\xf2V\xc2\x96\xb5]{\xbf+\xa0t\xbdA\xe6\xb8]\xf8=w!\x10\x18\xba\xc4\xd0\x05F\x08\xf4\x10\xc8\x10\x87k\xdc\x94\xec?N\xbaK%#\xc9\x85\xd6\xa5y\x1d\x01\xd1\xd0\xda\xce\xd6\x03\xc7y~\xcb\x1e\xee{\xa9\x88\xcf\x0f\x1c\x07\x9b\x05$\xdc~\xe5H6\xfd\xecFl\x16\x957\xff\xb6\x8f?\x94\x1e\xa0\xb8\xac\n\xb7\xd7\xc7=3G\xb3\xb7\xba\x89}\xc7\xdd\xa0\x08\x1a\x1b=\xfc\xad\xbc\x961\xe9Id\xa6\xf4\x03\xe0\xa3\x83\'\xc4\xddk\xff\xf8I\xd3\x9f\x7f~\x17\x00\xc3\xa7@\xc5\xf6S\x18\x89\xccF\x16\xaeZ\x8d\xc3\xf5+4\x9b\x99\xf7D\xab\xb3p7C\xd7M\xb5\x18\x9c\xcc\x98A\x89$\xba\x1ch\x9a\x8a\xdf\x1f\xa4\xb2\xbe\x85\xc3\r\xad\x94\xd7\x9d\xc4\xe3\xf1\x99\xe37! \xa43/w\x08E\xabo\xe6\xad\xd2\x03,_\xff)u\x8d\xad\x9d\x9ak\xd5x\xfb\xe1\x85\xe8Rr\xebs\x1ft\x89_:\xa9\x89\xb1\x8c\xcfL\x95zL\xb2\xd8{\xa0\xfcH\xdd\xde=3\xd8\xba\xeeX\xd71yw a\x9a\x98\xbb\xd4\x89#\xeeu\x9c\xf1\xb7D\x07\x95g0\xe2Nu\x92\x18\x86\xec=\xb35$)qN&g\xf5\xa7\xb0\xac\n]7:\xff\x1f6\xe8\xf4\xc4X\xfc\xbaNC\xab\xafg\x91\xd5/\r\xf4\x90\x87\xda\x03\xd7\xb2\xe97\x9f\x92\xb7\x02\xb6\x14\x9ca\x18\x1a\x19\xbbM\xbd\xd7E\xd2\xc0\xcd8\x13\xa6\x86=\x99\t\xe8l\x83\xa1\x94\x9d\x85\x90\xe8\x92\xe9\x1a\xf2\xf4\xb3\x94\x88\x87\x92\xd2L&#\x13_W\xaa@\xb5z8\xb2\xf3\xa7\x14\x16\xbc\xd2\xdbW{\xee\xb8y-\xe4\xad\x80\xd2\xd7<4\x1e\xcf\xa7\xad\xf9/\xe1\xacT\x84\xd99+\x10\x0e\xab\x85I\xc3\xd3\xb0j*\x04\xf5N\x17\xdb\x15DW\xc1u\x03\x02!\x92\\\x0e\xee\x9c>\n\x0cC\xa2j\x92\xb8\x01\x02\xc3h\xe1\xc8\x8eeQ\x10\xbd\xfc\xee\xab\xf7ah\xe56\xc8[\x0e\x9f\xbc\x10`p\xee\xfb\xe6\xcd\x8a\xe9\xa8\x96\xb3\xa3C\xc2\xfc\xdc!lxd!\xb3G\xa5S\xef\xf5QY{\x12:\xfc\xa6\xc0\xba\x0e!\xddL\xe5\x03A\xd0%\x99\x03\x13y`n.O\xdf5\x93E\x13\x87\xf1\xf4\xa62A|\x9a\xa0\xad\xe9\x10\xf5\xe5w\xb2\xb9`cd\x84@\xe1\xda^\x86\'gZ\x11c\x9a\xbfTA\xb1/\xc2\xee|\x1dG\\?\xa0g\x9c9\x85\x91\xac\xfe\xf1\xbc\xbfb\x119\xe9I\xf8\x02!Z;\x02\xec?\xee\xa6\xba\xa9\x15\xb7\xc7\x87aH\xfa\xc5X\xc9L\x89#;-\x9e\xfeqN\xec\x16\x8d\x17\x8bw\xcbU\x9b\xca\xc5I\xbf\x02\xf5\xe5\xef\xe3mZ\xcc\x96\xa7j\xba\xc9\xd3\xeb\x14\xe8\xdbV\xfe\n\x90\n\x14>\t\xb3\x7f\x9c\x8c3\xf65\x1cqW\xa1Y\x1d\xdd:~\xa7\x023\x0c\x84\x10\xac\xbcv\x12\x0f\xcc\x19K\x8a\xcb\x81USP\x15%j\xdf\xbaa\x10\xd4\r\xdcm~\xde\xff\xe20\xcf\x17\xef\xa5\xd2c\t\xe1on\xa2v\xdf\xe3\x14\xff\xf6w\xe1\xc1,l\xe9\xcb/\x1f\xba\xb1\xd3\x85\xd2\xf9\xcb\xe6a\x8fy\x00\x8bc\x06\x8a\x9a\x86\xaau\x02\xea\xda\x97\x95\x12\x02!\x90\x92!\xe9I\x0cK\x8d\'\xc6n&\x9e\x1d\x81\xa0lm\x0f\x88\x1ao\x90\xea\x9a\x16\xd3\xb8m\xf6]\xb4T\x7f\xc07\xbb\x9f\xe5\xcb\xbf\xb8\xcf\xa4J\xe7\x0f\x04\xe8\xe6\xf2.\x9b\xa90db.\xf6\x98\x19\x08\xe5\x1aTm\x0e\x8aEE\xb3t\x8f;]\xca\x80hW]\xb3\x85\xdf\x0b\x814\xaa\x11\xca\x16\x8c\xc0fj\x0el\xa7\xf4\xf5\xda\x1eg\x9d\xd5\x80\xf1|\xd6\xfceP\xf4\xb4\xf9|\xd6\x83\x02\xab\xd3\x81\xd5\x11\x83\x94\x93\x90\xc6,T\xcb8\x145\x1bEI\x02\xe1D\x08\x15)C\x08\xc5\x83P\xeb\x08u\xec#\x14\xdc\x89\xc5\xbe\x05\xa8\xa2\xb9\xa6\x83O^\n\xf4\xa5D\xf8\x7fM\xdd#\x1d\xf5i\xe0U\x00\x00\x00\x00IEND\xaeB`\x82' + fqfn = os.path.join(media_dir, 'image.png') + open(fqfn, 'wb').write(image_content) + + hashes = set() + h = utils.get_text_hash(image_content) + hashes.add(h); print h + h = utils.get_file_hash(fqfn) + hashes.add(h); print h + h = utils.get_text_hash(open(fqfn, 'rb').read()) + hashes.add(h); print h + h = utils.get_text_hash(open(fqfn, 'r').read()) + hashes.add(h); print h + print 'Hashes:', len(hashes) + + # Create test file. + self.assertEqual(len(hashes), 1) + image_content = u'aあä' + fqfn = os.path.join(media_dir, 'test.txt') + open(fqfn, 'wb').write(image_content.encode('utf-8')) + + hashes = set() + h = utils.get_text_hash(image_content) + hashes.add(h); print h + h = utils.get_file_hash(fqfn) + hashes.add(h); print h + h = utils.get_text_hash(open(fqfn, 'rb').read()) + hashes.add(h); print h + h = utils.get_text_hash(open(fqfn, 'r').read()) + hashes.add(h); print h + print 'Hashes:', len(hashes) + + self.assertEqual(len(hashes), 1) class DatabaseFilesViewTestCase(TestCase): fixtures = ['test_data.json'] diff --git a/database_files/utils.py b/database_files/utils.py index 8ee453b..665cfa4 100644 --- a/database_files/utils.py +++ b/database_files/utils.py @@ -5,6 +5,10 @@ from django.conf import settings +DEFAULT_ENFORCE_ENCODING = getattr(settings, 'DB_FILES_DEFAULT_ENFORCE_ENCODING', True) +DEFAULT_ENCODING = getattr(settings, 'DB_FILES_DEFAULT_ENCODING', 'ascii') +DEFAULT_ERROR_METHOD = getattr(settings, 'DB_FILES_DEFAULT_ERROR_METHOD', 'ignore') + def is_fresh(name, content_hash): """ Returns true if the file exists on the local filesystem and matches the @@ -45,20 +49,42 @@ def write_file(name, content, overwrite=False): if perms: os.system('chmod -R %s "%s"' % (perms, dirs)) -def get_file_hash(fin): +#def get_file_hash(fin): +# """ +# Iteratively builds a file hash without loading the entire file into memory. +# """ +# if isinstance(fin, basestring): +# fin = open(fin) +# h = hashlib.sha512() +# for text in fin.readlines(): +# if not isinstance(text, unicode): +# text = unicode(text, encoding='utf-8', errors='replace') +# h.update(text.encode('utf-8', 'replace')) +# return h.hexdigest() + +def get_file_hash(fin, + force_encoding=DEFAULT_ENFORCE_ENCODING, + encoding=DEFAULT_ENCODING, + errors=DEFAULT_ERROR_METHOD): """ Iteratively builds a file hash without loading the entire file into memory. """ if isinstance(fin, basestring): - fin = open(fin) + fin = open(fin, 'r') h = hashlib.sha512() - for text in fin.readlines(): - if not isinstance(text, unicode): - text = unicode(text, encoding='utf-8', errors='replace') - h.update(text.encode('utf-8', 'replace')) + while 1: + text = fin.read(1000) + if not text: + break + if force_encoding: + if not isinstance(text, unicode): + text = unicode(text, encoding=encoding, errors=errors) + h.update(text.encode(encoding, errors)) + else: + h.update(text) return h.hexdigest() -def get_text_hash(text): +def get_text_hash_0004(text): """ Returns the hash of the given text. """ @@ -68,4 +94,18 @@ def get_text_hash(text): h.update(text.encode('utf-8', 'replace')) return h.hexdigest() -get_text_hash_0004 = get_text_hash +def get_text_hash(text, + force_encoding=DEFAULT_ENFORCE_ENCODING, + encoding=DEFAULT_ENCODING, + errors=DEFAULT_ERROR_METHOD): + """ + Returns the hash of the given text. + """ + h = hashlib.sha512() + if force_encoding: + if not isinstance(text, unicode): + text = unicode(text, encoding=encoding, errors=errors) + h.update(text.encode(encoding, errors)) + else: + h.update(text) + return h.hexdigest() diff --git a/setup.py b/setup.py index 131a88a..74556a4 100644 --- a/setup.py +++ b/setup.py @@ -23,6 +23,7 @@ def finalize_options(self): pass def run(self): os.system('django-admin.py test --pythonpath=. --settings=database_files.tests.settings tests') + #os.system('django-admin.py test --pythonpath=. --settings=database_files.tests.settings tests.DatabaseFilesTestCase.test_hash') setup( name='django-database-files', From a9b396acf5612fa976d2d57d62589a993609dad3 Mon Sep 17 00:00:00 2001 From: chris Date: Tue, 21 Aug 2012 14:20:54 -0400 Subject: [PATCH 17/82] Fixed management command. --- database_files/management/commands/database_files_rehash.py | 1 - 1 file changed, 1 deletion(-) diff --git a/database_files/management/commands/database_files_rehash.py b/database_files/management/commands/database_files_rehash.py index d12bb63..ef305a7 100644 --- a/database_files/management/commands/database_files_rehash.py +++ b/database_files/management/commands/database_files_rehash.py @@ -20,7 +20,6 @@ class Command(BaseCommand): def handle(self, *args, **options): tmp_debug = settings.DEBUG settings.DEBUG = False - dryrun = options['dryrun'] try: q = File.objects.all() total = q.count() From 3b739685a4795f287483e2ce9889f93f9a3f40a2 Mon Sep 17 00:00:00 2001 From: chris Date: Wed, 19 Sep 2012 17:12:38 -0400 Subject: [PATCH 18/82] Added file hash caching on the filesystem. --- .../commands/database_files_dump.py | 30 ++--------------- database_files/manager.py | 5 ++- database_files/models.py | 32 ++++++++++++++++++- database_files/storage.py | 7 ++-- database_files/tests/tests.py | 32 +++++++++++++------ database_files/utils.py | 27 ++++++++++++++++ 6 files changed, 90 insertions(+), 43 deletions(-) diff --git a/database_files/management/commands/database_files_dump.py b/database_files/management/commands/database_files_dump.py index 2c10a03..36534c9 100644 --- a/database_files/management/commands/database_files_dump.py +++ b/database_files/management/commands/database_files_dump.py @@ -1,14 +1,9 @@ import os +from optparse import make_option -from django.conf import settings -from django.core.files.storage import default_storage from django.core.management.base import BaseCommand, CommandError -from django.db.models import FileField, ImageField from database_files.models import File -from database_files.utils import write_file, is_fresh - -from optparse import make_option class Command(BaseCommand): option_list = BaseCommand.option_list + ( @@ -21,24 +16,5 @@ class Command(BaseCommand): 'MEDIA_ROOT.' def handle(self, *args, **options): - tmp_debug = settings.DEBUG - settings.DEBUG = False - try: - q = File.objects.all().values_list('id', 'name', '_content_hash') - total = q.count() - i = 0 - for (file_id, name, content_hash) in q: - i += 1 - if not i % 100: - print '%i of %i' % (i, total) - if not is_fresh(name=name, content_hash=content_hash): - print 'File %i-%s is stale. Writing to local file system...' \ - % (file_id, name) - file = File.objects.get(id=file_id) - write_file( - file.name, - file.content, - overwrite=True) - finally: - settings.DEBUG = tmp_debug - \ No newline at end of file + File.dump_files(verbose=True) + \ No newline at end of file diff --git a/database_files/manager.py b/database_files/manager.py index feb3ae6..3ce79f5 100644 --- a/database_files/manager.py +++ b/database_files/manager.py @@ -3,6 +3,5 @@ class FileManager(models.Manager): def get_from_name(self, name): -# print 'name:',name -# return self.get(pk=os.path.splitext(os.path.split(name)[1])[0]) - return self.get(name=name) \ No newline at end of file + return self.get(name=name) + \ No newline at end of file diff --git a/database_files/models.py b/database_files/models.py index 6ea11f9..fef83b4 100644 --- a/database_files/models.py +++ b/database_files/models.py @@ -1,9 +1,11 @@ import base64 +from django.conf import settings from django.db import models from django.utils import timezone from database_files import utils +from database_files.utils import write_file, is_fresh from database_files.manager import FileManager class File(models.Model): @@ -61,4 +63,32 @@ def content_hash(self): if not self._content_hash and self._content: self._content_hash = utils.get_text_hash(self.content) return self._content_hash - \ No newline at end of file + + @classmethod + def dump_files(cls, debug=True, verbose=False): + if debug: + tmp_debug = settings.DEBUG + settings.DEBUG = False + try: + q = cls.objects.only('id', 'name', '_content_hash').values_list('id', 'name', '_content_hash') + total = q.count() + if verbose: + print 'Checking %i total files...' % (total,) + i = 0 + for (file_id, name, content_hash) in q: + i += 1 + if verbose and not i % 100: + print '%i of %i' % (i, total) + if not is_fresh(name=name, content_hash=content_hash): + if verbose: + print 'File %i-%s is stale. Writing to local file system...' \ + % (file_id, name) + file = File.objects.get(id=file_id) + write_file( + file.name, + file.content, + overwrite=True) + finally: + if debug: + settings.DEBUG = tmp_debug + \ No newline at end of file diff --git a/database_files/storage.py b/database_files/storage.py index b798f49..a0ffba6 100644 --- a/database_files/storage.py +++ b/database_files/storage.py @@ -7,7 +7,7 @@ from django.core.urlresolvers import reverse from database_files import models -from database_files.utils import write_file +from database_files import utils class DatabaseStorage(FileSystemStorage): @@ -67,7 +67,7 @@ def _save(self, name, content): ) # Automatically write the change to the local file system. if getattr(settings, 'DATABASE_FILES_FS_AUTO_WRITE', True): - write_file(name, content, overwrite=True) + utils.write_file(name, content, overwrite=True) #TODO:add callback to handle custom save behavior? return self._generate_name(name, f.pk) @@ -86,6 +86,9 @@ def delete(self, name): """ try: models.File.objects.get_from_name(name).delete() + hash_fn = utils.get_hash_fn(name) + if os.path.isfile(hash_fn): + os.remove(hash_fn) except models.File.DoesNotExist: pass return super(DatabaseStorage, self).delete(name) diff --git a/database_files/tests/tests.py b/database_files/tests/tests.py index 8eee539..a9af10d 100644 --- a/database_files/tests/tests.py +++ b/database_files/tests/tests.py @@ -13,6 +13,7 @@ DIR = os.path.abspath(os.path.split(__file__)[0]) class DatabaseFilesTestCase(TestCase): + def test_adding_file(self): # Create default thing storing reference to file @@ -67,6 +68,8 @@ def test_adding_file(self): t.upload.delete() self.assertEqual(File.objects.count(), 1) + File.dump_files() + # Confirm when delete a file from the database, we also delete it from # the filesystem. self.assertEqual(default_storage.exists('i/special/test.txt'), True) @@ -75,6 +78,7 @@ def test_adding_file(self): self.assertEqual(os.path.isfile(fqfn), False) def test_hash(self): + verbose = 0 # Create test file. media_dir = os.path.join(DIR, 'media/i/special') @@ -86,14 +90,18 @@ def test_hash(self): hashes = set() h = utils.get_text_hash(image_content) - hashes.add(h); print h + hashes.add(h) + if verbose: print h h = utils.get_file_hash(fqfn) - hashes.add(h); print h + hashes.add(h) + if verbose: print h h = utils.get_text_hash(open(fqfn, 'rb').read()) - hashes.add(h); print h + hashes.add(h) + if verbose: print h h = utils.get_text_hash(open(fqfn, 'r').read()) - hashes.add(h); print h - print 'Hashes:', len(hashes) + hashes.add(h) + if verbose: print h + #print 'Hashes:', len(hashes) # Create test file. self.assertEqual(len(hashes), 1) @@ -103,14 +111,18 @@ def test_hash(self): hashes = set() h = utils.get_text_hash(image_content) - hashes.add(h); print h + hashes.add(h) + if verbose: print h h = utils.get_file_hash(fqfn) - hashes.add(h); print h + hashes.add(h) + if verbose: print h h = utils.get_text_hash(open(fqfn, 'rb').read()) - hashes.add(h); print h + hashes.add(h) + if verbose: print h h = utils.get_text_hash(open(fqfn, 'r').read()) - hashes.add(h); print h - print 'Hashes:', len(hashes) + hashes.add(h) + if verbose: print h + #print 'Hashes:', len(hashes) self.assertEqual(len(hashes), 1) diff --git a/database_files/utils.py b/database_files/utils.py index 665cfa4..b221314 100644 --- a/database_files/utils.py +++ b/database_files/utils.py @@ -8,6 +8,7 @@ DEFAULT_ENFORCE_ENCODING = getattr(settings, 'DB_FILES_DEFAULT_ENFORCE_ENCODING', True) DEFAULT_ENCODING = getattr(settings, 'DB_FILES_DEFAULT_ENCODING', 'ascii') DEFAULT_ERROR_METHOD = getattr(settings, 'DB_FILES_DEFAULT_ERROR_METHOD', 'ignore') +DEFAULT_HASH_FN_TEMPLATE = getattr(settings, 'DB_FILES_DEFAULT_HASH_FN_TEMPLATE', '%s.hash') def is_fresh(name, content_hash): """ @@ -16,6 +17,13 @@ def is_fresh(name, content_hash): """ if not content_hash: return False + + # Check for cached hash file. + hash_fn = get_hash_fn(name) + if os.path.isfile(hash_fn): + return open(hash_fn).read().strip() == content_hash + + # Otherwise, calculate the hash of the local file. fqfn = os.path.join(settings.MEDIA_ROOT, name) fqfn = os.path.normpath(fqfn) if not os.path.isfile(fqfn): @@ -23,6 +31,20 @@ def is_fresh(name, content_hash): local_content_hash = get_file_hash(fqfn) return local_content_hash == content_hash +def get_hash_fn(name): + """ + Returns the filename for the hash file. + """ + fqfn = os.path.join(settings.MEDIA_ROOT, name) + fqfn = os.path.normpath(fqfn) + dirs,fn = os.path.split(fqfn) + if not os.path.isdir(dirs): + os.makedirs(dirs) + fqfn_parts = os.path.split(fqfn) + hash_fn = os.path.join(fqfn_parts[0], + DEFAULT_HASH_FN_TEMPLATE % fqfn_parts[1]) + return hash_fn + def write_file(name, content, overwrite=False): """ Writes the given content to the relative filename under the MEDIA_ROOT. @@ -36,6 +58,11 @@ def write_file(name, content, overwrite=False): os.makedirs(dirs) open(fqfn, 'wb').write(content) + # Cache hash. + hash = get_file_hash(fqfn) + hash_fn = get_hash_fn(name) + open(hash_fn, 'wb').write(hash) + # Set ownership and permissions. uname = getattr(settings, 'DATABASE_FILES_USER', None) gname = getattr(settings, 'DATABASE_FILES_GROUP', None) From 57fe6c28fcecb6fc2c63d025611d441a02832a20 Mon Sep 17 00:00:00 2001 From: chris Date: Wed, 19 Sep 2012 17:13:12 -0400 Subject: [PATCH 19/82] Updated version. --- database_files/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/database_files/__init__.py b/database_files/__init__.py index 3bf2ad9..8a34fe9 100644 --- a/database_files/__init__.py +++ b/database_files/__init__.py @@ -1,2 +1,2 @@ -VERSION = (0, 1, 8) +VERSION = (0, 1, 9) __version__ = '.'.join(map(str, VERSION)) \ No newline at end of file From aaa95c6e447f1f215c85c6fa99facabab6dd6e1c Mon Sep 17 00:00:00 2001 From: chris Date: Fri, 21 Sep 2012 18:29:43 -0400 Subject: [PATCH 20/82] Modified the default behavior to automatically export files from the database if they don't exist on the filesystem. --- database_files/storage.py | 11 +++++++++++ database_files/tests/tests.py | 21 +++++++++++++++------ database_files/utils.py | 6 ++++++ setup.py | 2 +- 4 files changed, 33 insertions(+), 7 deletions(-) diff --git a/database_files/storage.py b/database_files/storage.py index a0ffba6..da5d042 100644 --- a/database_files/storage.py +++ b/database_files/storage.py @@ -9,6 +9,8 @@ from database_files import models from database_files import utils +AUTO_EXPORT_DB_TO_FS = getattr(settings, 'DB_FILES_AUTO_EXPORT_DB_TO_FS', True) + class DatabaseStorage(FileSystemStorage): def _generate_name(self, name, pk): @@ -29,6 +31,15 @@ def _open(self, name, mode='rb'): f = models.File.objects.get_from_name(name) content = f.content size = f.size + if AUTO_EXPORT_DB_TO_FS \ + and not utils.is_fresh(f.name, f.content_hash): + # Automatically write the file to the filesystem + # if it's missing and exists in the database. + # This happens if we're using multiple web servers connected + # to a common databaes behind a load balancer. + # One user might upload a file from one web server, and then + # another might access if from another server. + utils.write_file(f.name, f.content) except models.File.DoesNotExist: # If not yet in the database, check the local file system # and load it into the database if present. diff --git a/database_files/tests/tests.py b/database_files/tests/tests.py index a9af10d..45871cd 100644 --- a/database_files/tests/tests.py +++ b/database_files/tests/tests.py @@ -21,10 +21,11 @@ def test_adding_file(self): media_dir = os.path.join(DIR,'media/i/special') if not os.path.isdir(media_dir): os.makedirs(media_dir) - fqfn = os.path.join(media_dir,'test.txt') - open(fqfn,'w').write('hello there') - o = Thing() - o.upload = 'i/special/test.txt' + test_fqfn = os.path.join(media_dir,'test.txt') + open(test_fqfn,'w').write('hello there') + o1 = o = Thing() + test_fn = 'i/special/test.txt' + o.upload = test_fn o.save() id = o.id @@ -32,7 +33,7 @@ def test_adding_file(self): Thing.objects.update() q = Thing.objects.all() self.assertEqual(q.count(), 1) - self.assertEqual(q[0].upload.name, 'i/special/test.txt') + self.assertEqual(q[0].upload.name, test_fn) # Confirm the file only exists on the file system # and hasn't been loaded into the database. @@ -68,6 +69,14 @@ def test_adding_file(self): t.upload.delete() self.assertEqual(File.objects.count(), 1) + # Delete file from local filesystem and re-export it from the database. + self.assertEqual(os.path.isfile(test_fqfn), True) + os.remove(test_fqfn) + self.assertEqual(os.path.isfile(test_fqfn), False) + o1.upload.read() # This forces the re-export to the filesystem. + self.assertEqual(os.path.isfile(test_fqfn), True) + + # This dumps all files to the filesystem. File.dump_files() # Confirm when delete a file from the database, we also delete it from @@ -75,7 +84,7 @@ def test_adding_file(self): self.assertEqual(default_storage.exists('i/special/test.txt'), True) default_storage.delete('i/special/test.txt') self.assertEqual(default_storage.exists('i/special/test.txt'), False) - self.assertEqual(os.path.isfile(fqfn), False) + self.assertEqual(os.path.isfile(test_fqfn), False) def test_hash(self): verbose = 0 diff --git a/database_files/utils.py b/database_files/utils.py index b221314..488dedd 100644 --- a/database_files/utils.py +++ b/database_files/utils.py @@ -18,6 +18,12 @@ def is_fresh(name, content_hash): if not content_hash: return False + # Check that the actual file exists. + fqfn = os.path.join(settings.MEDIA_ROOT, name) + fqfn = os.path.normpath(fqfn) + if not os.path.isfile(fqfn): + return False + # Check for cached hash file. hash_fn = get_hash_fn(name) if os.path.isfile(hash_fn): diff --git a/setup.py b/setup.py index 74556a4..d7b5cfd 100644 --- a/setup.py +++ b/setup.py @@ -23,7 +23,7 @@ def finalize_options(self): pass def run(self): os.system('django-admin.py test --pythonpath=. --settings=database_files.tests.settings tests') - #os.system('django-admin.py test --pythonpath=. --settings=database_files.tests.settings tests.DatabaseFilesTestCase.test_hash') + #os.system('django-admin.py test --pythonpath=. --settings=database_files.tests.settings tests.DatabaseFilesTestCase.test_adding_file') setup( name='django-database-files', From b407fa467850576c7744f121a303d7d2423fa128 Mon Sep 17 00:00:00 2001 From: chris Date: Fri, 21 Sep 2012 18:30:04 -0400 Subject: [PATCH 21/82] Updated version number. --- database_files/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/database_files/__init__.py b/database_files/__init__.py index 8a34fe9..82d20ff 100644 --- a/database_files/__init__.py +++ b/database_files/__init__.py @@ -1,2 +1,2 @@ -VERSION = (0, 1, 9) +VERSION = (0, 1, 10) __version__ = '.'.join(map(str, VERSION)) \ No newline at end of file From 76dfde2db36cc460e9a76e11c6292a1007075575 Mon Sep 17 00:00:00 2001 From: chris Date: Wed, 14 Nov 2012 11:15:22 -0500 Subject: [PATCH 22/82] Modified file dump command to update hash upon write and allow targeting of specific files. --- database_files/__init__.py | 2 +- .../management/commands/database_files_rehash.py | 9 ++++++--- database_files/models.py | 4 +++- 3 files changed, 10 insertions(+), 5 deletions(-) diff --git a/database_files/__init__.py b/database_files/__init__.py index 82d20ff..fec5954 100644 --- a/database_files/__init__.py +++ b/database_files/__init__.py @@ -1,2 +1,2 @@ -VERSION = (0, 1, 10) +VERSION = (0, 1, 11) __version__ = '.'.join(map(str, VERSION)) \ No newline at end of file diff --git a/database_files/management/commands/database_files_rehash.py b/database_files/management/commands/database_files_rehash.py index ef305a7..ad11b21 100644 --- a/database_files/management/commands/database_files_rehash.py +++ b/database_files/management/commands/database_files_rehash.py @@ -6,8 +6,9 @@ from database_files.models import File class Command(BaseCommand): - args = '' - help = 'Regenerates hashes for all files.' + args = ' ... ' + help = 'Regenerates hashes for files. If no filenames given, ' + \ + 'rehashes everything.' option_list = BaseCommand.option_list + ( # make_option('--dryrun', # action='store_true', @@ -22,9 +23,11 @@ def handle(self, *args, **options): settings.DEBUG = False try: q = File.objects.all() + if args: + q = q.filter(name__in=args) total = q.count() i = 1 - for f in q: + for f in q.iterator(): print '%i of %i: %s' % (i, total, f.name) f._content_hash = None f.save() diff --git a/database_files/models.py b/database_files/models.py index fef83b4..67be186 100644 --- a/database_files/models.py +++ b/database_files/models.py @@ -75,7 +75,7 @@ def dump_files(cls, debug=True, verbose=False): if verbose: print 'Checking %i total files...' % (total,) i = 0 - for (file_id, name, content_hash) in q: + for (file_id, name, content_hash) in q.iterator(): i += 1 if verbose and not i % 100: print '%i of %i' % (i, total) @@ -88,6 +88,8 @@ def dump_files(cls, debug=True, verbose=False): file.name, file.content, overwrite=True) + file._content_hash = None + file.save() finally: if debug: settings.DEBUG = tmp_debug From 816f3eaec154ac9b8d6e3960df5e45505f025323 Mon Sep 17 00:00:00 2001 From: chris Date: Thu, 9 May 2013 10:42:23 -0400 Subject: [PATCH 23/82] Added configurable file URL constructor. --- database_files/models.py | 2 ++ database_files/settings.py | 36 ++++++++++++++++++++++++++++++++++++ database_files/storage.py | 3 +-- 3 files changed, 39 insertions(+), 2 deletions(-) create mode 100644 database_files/settings.py diff --git a/database_files/models.py b/database_files/models.py index 67be186..639eef0 100644 --- a/database_files/models.py +++ b/database_files/models.py @@ -1,5 +1,7 @@ import base64 +import settings as _settings + from django.conf import settings from django.db import models from django.utils import timezone diff --git a/database_files/settings.py b/database_files/settings.py new file mode 100644 index 0000000..5c0fa1b --- /dev/null +++ b/database_files/settings.py @@ -0,0 +1,36 @@ +import os + +from django.conf import settings +from django.core.urlresolvers import reverse + +def URL_METHOD_1(name): + """ + Construct file URL based on media URL. + """ + return os.path.join(settings.MEDIA_URL, name) + +def URL_METHOD_2(name): + """ + Construct file URL based on configured URL pattern. + """ + return reverse('database_file', kwargs={'name': name}) + +URL_METHODS = ( + ('URL_METHOD_1', URL_METHOD_1), + ('URL_METHOD_2', URL_METHOD_2), +) + +DATABASE_FILES_URL_METHOD_NAME = getattr( + settings, + 'DATABASE_FILES_URL_METHOD', + 'URL_METHOD_1') + +if callable(DATABASE_FILES_URL_METHOD_NAME): + method = DATABASE_FILES_URL_METHOD_NAME +else: + method = dict(URL_METHODS)[DATABASE_FILES_URL_METHOD_NAME] + +DATABASE_FILES_URL_METHOD = setattr( + settings, + 'DATABASE_FILES_URL_METHOD', + method) diff --git a/database_files/storage.py b/database_files/storage.py index da5d042..56a06c1 100644 --- a/database_files/storage.py +++ b/database_files/storage.py @@ -108,8 +108,7 @@ def url(self, name): """ Returns the web-accessible URL for the file with filename `name`. """ - return os.path.join(settings.MEDIA_URL, name) - #return reverse('database_file', kwargs={'name': name}) + return settings.DATABASE_FILES_URL_METHOD(name) def size(self, name): """ From 40fd4cfb2c540a1591bc80d313401e1de4d5877b Mon Sep 17 00:00:00 2001 From: chris Date: Thu, 9 May 2013 10:42:39 -0400 Subject: [PATCH 24/82] Updated version. --- database_files/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/database_files/__init__.py b/database_files/__init__.py index fec5954..172fe9a 100644 --- a/database_files/__init__.py +++ b/database_files/__init__.py @@ -1,2 +1,2 @@ -VERSION = (0, 1, 11) +VERSION = (0, 1, 12) __version__ = '.'.join(map(str, VERSION)) \ No newline at end of file From 882b9a0098db1dbecd9241be2e500fc5b81ddc79 Mon Sep 17 00:00:00 2001 From: chris Date: Tue, 16 Jul 2013 11:11:10 -0400 Subject: [PATCH 25/82] Improved performance of cleanup command. --- database_files/__init__.py | 2 +- .../commands/database_files_cleanup.py | 22 +++++++++++++++---- 2 files changed, 19 insertions(+), 5 deletions(-) diff --git a/database_files/__init__.py b/database_files/__init__.py index 172fe9a..279ccfe 100644 --- a/database_files/__init__.py +++ b/database_files/__init__.py @@ -1,2 +1,2 @@ -VERSION = (0, 1, 12) +VERSION = (0, 1, 13) __version__ = '.'.join(map(str, VERSION)) \ No newline at end of file diff --git a/database_files/management/commands/database_files_cleanup.py b/database_files/management/commands/database_files_cleanup.py index ec18fd6..daa2138 100644 --- a/database_files/management/commands/database_files_cleanup.py +++ b/database_files/management/commands/database_files_cleanup.py @@ -28,13 +28,20 @@ def handle(self, *args, **options): dryrun = options['dryrun'] try: for model in get_models(): - for field in model._meta.fields: + print 'Checking model %s...' % (model,) + for field in model._meta.fields: if not isinstance(field, (FileField, ImageField)): continue # Ignore records with null or empty string values. q = {'%s__isnull'%field.name:False} xq = {field.name:''} - for row in model.objects.filter(**q).exclude(**xq): + subq = model.objects.filter(**q).exclude(**xq) + subq_total = subq.count() + subq_i = 0 + for row in subq.iterator(): + subq_i += 1 + if subq_i == 1 or not subq_i % 100: + print '%i of %i' % (subq_i, subq_total) file = getattr(row, field.name) if file is None: continue @@ -42,9 +49,16 @@ def handle(self, *args, **options): continue names.add(file.name) # Find all database files with names not in our list. - orphan_files = File.objects.exclude(name__in=names) + print 'Finding orphaned files...' + orphan_files = File.objects.exclude(name__in=names).only('name', 'size') total_bytes = 0 - for f in orphan_files: + orphan_total = orphan_files.count() + orphan_i = 0 + print 'Deleting %i orphaned files...' % (orphan_total,) + for f in orphan_files.iterator(): + orphan_i += 1 + if orphan_i == 1 or not orphan_i % 100: + print '%i of %i' % (orphan_i, orphan_total) total_bytes += f.size if dryrun: print 'File %s is orphaned.' % (f.name,) From 1ed4ad79a10bd146a5765c9165538957d35eef14 Mon Sep 17 00:00:00 2001 From: chris Date: Wed, 28 Aug 2013 17:12:34 -0400 Subject: [PATCH 26/82] Added view to automatically handle retrieving files from either the filesystem or database. --- database_files/models.py | 15 +++++++++++++++ database_files/views.py | 18 ++++++++++++++++++ 2 files changed, 33 insertions(+) diff --git a/database_files/models.py b/database_files/models.py index 639eef0..df32f95 100644 --- a/database_files/models.py +++ b/database_files/models.py @@ -66,8 +66,23 @@ def content_hash(self): self._content_hash = utils.get_text_hash(self.content) return self._content_hash + def dump(self, check_hash=False): + """ + Writes the file content to the filesystem. + """ + write_file( + self.name, + self.content, + overwrite=True) + if check_hash: + self._content_hash = None + self.save() + @classmethod def dump_files(cls, debug=True, verbose=False): + """ + Writes all files to the filesystem. + """ if debug: tmp_debug = settings.DEBUG settings.DEBUG = False diff --git a/database_files/views.py b/database_files/views.py index 1b0b977..7946b11 100644 --- a/database_files/views.py +++ b/database_files/views.py @@ -4,6 +4,7 @@ from django.http import Http404, HttpResponse from django.shortcuts import get_object_or_404 from django.views.decorators.cache import cache_control +from django.views.static import serve as django_serve import mimetypes @@ -11,8 +12,25 @@ @cache_control(max_age=86400) def serve(request, name): + """ + Retrieves the file from the database. + """ f = get_object_or_404(File, name=name) + f.dump() mimetype = mimetypes.guess_type(name)[0] or 'application/octet-stream' response = HttpResponse(f.content, mimetype=mimetype) response['Content-Length'] = f.size return response + +def serve_mixed(request, path, document_root): + """ + First attempts to serve the file from the filesystem, + then tries the database. + """ + try: + # First attempt to serve from filesystem. + return django_serve(request, path, document_root) + except Http404: + # Then try serving from database. + return serve(request, path) + \ No newline at end of file From 44fed3c20c769e5abb6cac87e9994397f6469b9b Mon Sep 17 00:00:00 2001 From: chris Date: Wed, 28 Aug 2013 17:18:16 -0400 Subject: [PATCH 27/82] Updated version. --- database_files/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/database_files/__init__.py b/database_files/__init__.py index 279ccfe..d4ef6ae 100644 --- a/database_files/__init__.py +++ b/database_files/__init__.py @@ -1,2 +1,2 @@ -VERSION = (0, 1, 13) +VERSION = (0, 1, 14) __version__ = '.'.join(map(str, VERSION)) \ No newline at end of file From a08a2ab5b1783093e77c06df8574b8eeb8ee6cf8 Mon Sep 17 00:00:00 2001 From: chris Date: Mon, 28 Apr 2014 15:28:19 -0400 Subject: [PATCH 28/82] Fixed bug causing file to be unnecessarily re-written on every request. --- database_files/__init__.py | 2 +- database_files/models.py | 4 ++++ 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/database_files/__init__.py b/database_files/__init__.py index d4ef6ae..868a985 100644 --- a/database_files/__init__.py +++ b/database_files/__init__.py @@ -1,2 +1,2 @@ -VERSION = (0, 1, 14) +VERSION = (0, 2, 0) __version__ = '.'.join(map(str, VERSION)) \ No newline at end of file diff --git a/database_files/models.py b/database_files/models.py index df32f95..d734db6 100644 --- a/database_files/models.py +++ b/database_files/models.py @@ -69,7 +69,11 @@ def content_hash(self): def dump(self, check_hash=False): """ Writes the file content to the filesystem. + + If check_hash is true, clears the stored file hash and recalculates. """ + if is_fresh(self.name, self._content_hash): + return write_file( self.name, self.content, From 9580deaafa74d02f586c02df1a38c511b2d284b2 Mon Sep 17 00:00:00 2001 From: chris Date: Mon, 28 Apr 2014 16:11:45 -0400 Subject: [PATCH 29/82] Added table name. --- database_files/models.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/database_files/models.py b/database_files/models.py index d734db6..2445cfa 100644 --- a/database_files/models.py +++ b/database_files/models.py @@ -39,6 +39,9 @@ class File(models.Model): max_length=128, blank=True, null=True) + class Meta: + db_table = 'database_files_file' + def save(self, *args, **kwargs): # Check for and clear old content hash. From 93ebddf264139516578026594d23d7ba2748792c Mon Sep 17 00:00:00 2001 From: chris Date: Mon, 28 Apr 2014 17:33:35 -0400 Subject: [PATCH 30/82] Updated ignore. --- .gitignore | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.gitignore b/.gitignore index 6b1a720..9a2ec7c 100644 --- a/.gitignore +++ b/.gitignore @@ -2,4 +2,5 @@ /build .project .pydevproject -.settings \ No newline at end of file +.settings +/dist From 15c74992973b3c24e2a4fd1b545ffc7f19f306e4 Mon Sep 17 00:00:00 2001 From: chris Date: Wed, 18 Jun 2014 15:41:44 -0400 Subject: [PATCH 31/82] Added test image Python markup. --- database_files/tests/fixtures/test_image.txt | 1 + 1 file changed, 1 insertion(+) create mode 100644 database_files/tests/fixtures/test_image.txt diff --git a/database_files/tests/fixtures/test_image.txt b/database_files/tests/fixtures/test_image.txt new file mode 100644 index 0000000..3fe231d --- /dev/null +++ b/database_files/tests/fixtures/test_image.txt @@ -0,0 +1 @@ +\x89PNG\r\n\x1a\n\x00\x00\x00\rIHDR\x00\x00\x002\x00\x00\x009\x08\x06\x00\x00\x00t\xf8xr\x00\x00\x00\x01sRGB\x00\xae\xce\x1c\xe9\x00\x00\x00\x06bKGD\x00\xff\x00\xff\x00\xff\xa0\xbd\xa7\x93\x00\x00\x00\tpHYs\x00\x00\x1e\xc2\x00\x00\x1e\xc2\x01n\xd0u>\x00\x00\x00\x07tIME\x07\xdc\x08\x15\r0\x02\xc7\x0f^\x14\x00\x00\x0eNIDATh\xde\xd5\x9aytTU\x9e\xc7?\xf7\xbdWkRd\x0f\x01\x12\x02!\xc8\x120,\r6\xb2#\x10\xa2(\xeeK\xdb:n\xd3\xca\xd0\xb46\x8b\xf6\xc09=\xd3v\xdb\xa7\x91\xb8\xb5L\xab\xe3\xb8\xb5\x8c\xbd\x88\x83\xda-\x84\xc4\xb84\x13\x10F\x01\r\xb2%\x01C \tY\x8a$UIj{\xef\xce\x1f\xaf\xaa\x92\x90\x80@\xc0\xa6\xef9uRU\xa9\xba\xf7\xf7\xbd\xdf\xdf\xfe+\xb8Xk\xc1\x8aD\xe6/K\x03 \x7f%\x17{)\x17kcUU/s%\xa6\xbce[\xf8h6\x9b\xd7B\xfe\x8a\x7fL \xe9)I\t\xd3\'O\x9c-\xe2\x07\xfc\x89\x9c\xd9N6\x17\\Tf.\x1a\x90\xe1\x19\xa9\xa3\xe6\x8d\x19\x84\x96>f\x02#\xe6l\x02`\xf3ZX\xb0\xe2\x1f\x08\xc8\x84\xfb\xfb\xa5\xc6\xc7.\x9c;&\x1d\xabE\x95d\x8c\x9f\xc1\r\xbfx\x95\xf4\\\x85\xc2\x82\x8b\x02F\xb9\x80\xc6M\xcc\xf5\xab\x01p\x0eI\x9f1kl\xd6\xeca)\xfd\xb0)\x8a@\xb3\n\x06\xe4\xdc\xcb\x15\xb7\xad\xe1\xca\x1f:L0\xcb\xb9\xf4V\xd7\x1b\xce\xbec\xfe\xea7\x8bO\x18R\xca\x0f\xcb\xdd\x86kU\x89d\xd9\x16\x83\xe5E\x92%\xefHnyr#\xdf\xbb%\xbd\xf3\xbb\x17\x06\x90\xda\xe7\x1d\xf2W@a\x81\xf9|\xe6OV\xad\x7fz\xc9\xb3?\xbe\xe6\xfbI\xfb\x1b\xda\xe5\xafK\xbe\x11_\xd7y\x91\x08\x01H4\x9b\xc0\x95<\x12W\xf2\x1d$\x0fm\xe6\xc8\xce\xddTl7/\xa2b[\x9f\xc4\x10\xe7\xcd@D\xf8\xa1\xb7*\xd6\x9c\xac\x01\xb9C\x07\xbc\xf8\xdc\xe2E\x0bs\xb2\xd2)>\xe4f\xcb\xc1&^\xdfy\x8c\xa0n`\xe2\x08/)%B\x08:Z\xe1D\xf9\x0e\x02\xde\xd5H\xb6q\xb2\xdaO\xe9\xef\x8d\xef\x96\x91\xc8\xed\r\xbfcX\xde\xeds\xef\xbf\xe1\xca\xb1\x7f~n\xf1\xf5cGe\xa4\xcaV\x7f\x88=5mB\x02\xfb\xeb\xdb\xf0\xfaC\xdd\x81\x880;\x16\xbb ~`:V\xe7\xdd(\xea-\xc4$\xda\xc9\xc8m\xa1b[C\xf4\xb2\xce\x81%q.*\xe4Tm\xb4\xff\xf5\t`J\xf2e\xf7\xdf\xf0\xf0\xe2k\xa7\xdc>m\xf4\x90\xec\xef\rO\x07\x90\x80\x08\xea\x06\xaf}QGY\x8d\x87\xff\xfa\xec\x18\xbe\xa0\xde\x1dH\'3 \x84\x0c\x8b \xf0{\xc1\xd7Z\x83\xbbz\x03u\x07\xd6\xb2m\xfd1~\xf0[p\x1f\xedd\xbf\xcf@\xf2W\x9a1\x00`\xd2\x8f\xee[ro\xfe\x13\x8f\xdd<+e@\xa2K\xd5T5\n\x02\xe0\xbd\xaf\x1b\xa9h\xea\xe0\xc5\xd2\xa3T4\xb4\x99\xef\x8a3\x1c\x13\x05\x84@J0t\x89\xbb\xba\x9e\xe3e\x8fQ\xb2\xee\r\xf3\xfc\xe5\xb0\xf9\xa9>\x00\xe9\n`\xce\xc3\xc9C3\x07\xbc\xf9\xc6\xb2[\x17L\x1f3T\x86\xd5]tj\x0b|\\\xd9\xcc\xd7\'\xdax\xb7\xac\x8e\x92\xfd\x8d\xa0\x9d\x87w7\x81A[3\xd4\xee[G[\xd3c|\xf8|;\xf9\x8f\xc2\xe6\'\xcf\xc3F\xba\x80p,Z5\xfb\x9f\x17^\xf9\xde\x7f?z\xc7\xe4Q\x83S\x91R\n\x11\x96^\x08AP\x97|v\xb4\x95/k\xbd|x\xa8\x91\xc2\xb2z\xb0\x9c\xa7C\x8c\xb0gs\x803~20\x82\xd4\xecR\x8a\x9f\xf5\x90\xff(T\x94\x9e\x03\x90\xaeL,X\xb1d\xdd\x92\x1b^\\~\xe3\x8cA\t\xb1\x0e\xd9\x95E!\x04\xed\x01\x9d\x8f\x0f7SV\xd7\xc6\xa6\x03\rl\xda\xd7\x00\xea\x05\x88\xb3RJ\xacvp\xb8r\xd0\xacS\x89M\xfa\x94\x8f_p\x9f\xce\t\xa8\xbd\xc6\x85\xcda\xe3\x9a\xb1\xf4\xf1\xa2\xb5K~y\xf3\xb4\xb1\xb16\x8b&\xc3L\x10yT7\xfb\xd8t\xc0MM\xab\x9f\xff\xf9\xea\x04\x1f\x957u\xbf\xd5>\x05\x06a\xda\x8cj\x01g|\x06\xd6\x98\xd98b?\xe0\xd3\x97[X\xb0\x1c*\xb6\x7f\x0b\x90\x08\xda\xbc\xe5K\x8b\n\x96<>o|\xb6\x85\x88\xc9\x86=g\xc8\x80]\xc7\xbd|\\\xd9\x8c\xc7\xa7\xf3\xf2\xf6j>\xafj\x06E\\\x18\x10\xdd]\xb5\x00!q\xc6\xf5\xc7\xea\x1cMB\xfa\x07|\xf4\x1f\xbeS\xc1\xa8=\x02]\xc56\x1c\x8bV\xcd}\xe6\xa1\xeb_\xbcm\xfa\xe51\x11\x00B\x08tCR\xe7\t\xf0Ie3\xbbk\xbcT6\xb5\xf3\xfc\xd6*\xaa\xdd\x1d\x17F\x9d\xce\x04HQA\xb3g#uH\xc9,\xc1\xea\xeaf/j7\xbb(,@,X\x9e\xb4h\xfa\xc4\xf7V\xdf6{\x80\xd3\xa6\xc9\x88Q\xd7\xb4\xf8\xf9\xbfc\x1evV{\xa8hj\xa7p\x7f\x03\x1b\xbf:A{P\x07\xe5"\x82\xe8b5Xl\x02_\xcb4B\xa1=\x14\xae=\xc0\x82\x95Q\r\xd2\xba~\x0e\xc0\xe9\x8a{\xfd\xba9S\x86%\xc4\xd8$ \xaaN\xfa\xf8\xaa\xd6K\xbd7H\xc8\x90\x94\x947Rt\xb0\x89\x96\x8eP\xf4;\xdf\xd1\x12\x08Eb\x8b\x15\xd8b~\xcfU\xff\x92I\xe1\xda\x93\x91t\xa9\xbbB\xcfX|\xfd\xac\xebn\xda\xf8\x83\xe99RS\xa4\xe8\x08\x1a\x84\x0cS\xd8\xc3M\xed\xbc\xf5E\r\r\xde\xc0\x85\xb5\x83s\x89/\x8a\nMU\x12_\xb3\xa0\xa5\xfe\x1d6\xad\xb9\xd94\x89\x95(]\xd2h\x8d\xb4\xe1\x05\xce\x84\x14\xfc\xc1\x90hj\x0bR\xef\r\xf0u\xad\x97u[\xabx\xee\xa3\xc34x\xfeN \xa2\x9eP\x824L\xe3w\xc4/`\xfe#SM\x90\x06\x1a"\xac\xdfW?v\x1f\xfd\xb32>)o\x94\r\x1e\x9f\xf0\xeb\x065->\x1a[\xfcf\xf9\xa5*\x7f?\x10\x91e\x18 u\x90R`\xb1\xc4`s=H\xca\x88mly\xcaL1\x18\xb70\x86\xb1W\xbfM\xf2\xd0|\x84\xd2\xa9\xfa\x82\x0b\x17\x17.\xc4\n\x05\xa0\xe9\x1b\x08\xf9\xcd\xd7A\xdfQ<\x8d\x0b)z\xa6\xcc\xa4\xa3\xff\x8818\xe2\'\xa1\xa8\xa6\xd0\x8a\xe8\x8c\t\x97\n\x08\x80\x90\x0f\x8c`\xe7\xe5*\xea`l\xb1S\x99\xf6O(dM\x82\xf8\x81\x13\xd0\xac\xc9\\\xaaKJ\xf3\xe1o\x03#Z\x16HT\x0b(\xcau8\x13\x1d\nC&\xd9\xd1,Wa\x8b\xe5\xbb\xf6\xa7\xe7d\xe8z\x10\xda\x9b;\xf5]J\x11f\xe5*\xac\x0e\xa7B\xbf\xfe6T\xeb,T\x15$\xe2\x92d\x03\x01\xdeFS\xad"\xaa\x1eaE\xb3X\x91r\xa2\x82\xaa%a\xb1\'\xf5\xa1\x82\xbf\xf8l\xf8\xbd\xe0m \xeaa\xbb\x06I\xd5\x02\xd2\x98\xa1\xa0\x07Ga\x8b\x01y\x89\xa9\x95\x0c\x8b\xa3\x07\xc0]\xd5\x1b\x88p\x11\xa6\x82\xa2\xe6*\x18F&\x16; \xc5%i\x17\xee\xea\xae\x06\xde{\x11\xa6hY\nR&\xa3Z/AOe@s\r\x04\xda\xbf5\x97DQ\x924\x90j\x9fc\x85\x94=\xf6\xeefo\xe7\xb2\x7f\xa4f?Y\r\xbe\x96p9"z\xdf7\x9a\xc3k\x0e\xed\xbc-\xdc\x90(\xaa`p\x82\x9d\x1bF\xa72wD"\xd9\xc9N\x12\x9d\x164E\xe1PC\x1b\x7f\xfd\xba\x9e\re\r|s\xb2\x83\x0e\x9fn\xa6:\xa7o\ru2\xe1>\n\x1d\x9e\xce\xf2 |Qi\t1\xf4\xb3\xdbh\x0f\x049\xd1\xd2N0\xa8\x9b\x81\x1bEj\x08\xbc\xe8!P\xb5\xb3\x07\xa1\x1b$\xbbl,\x9d\x96\xc1\xe2\xa9\x19\xa4\xc4\xf4T\xcd\xc9\x83\xe3\x98<8\x8eU\xf3\x86\xf1\xa7\xdd\xb5\xbc\xbc\xa3\x86\xd2\n\xb7Y\x01\x9d\n&\xf2:\xe8\x83\xe6\xe3\x10h\xeb\x04\xe1\x0f\x92\x96\x1a\xc7\x8f\xe6\x8c\xe5\xa6+\x863b`"\xc7\xdc^6|v\x88\x7f\xdb\xb0\x1d\x7f0\x04\xe0S\x199{(\xae\x94\x1b\xd1\xac\xf2\xac\x1c\xb0\x94\x0cJp\xb0\xf1\x9e\\~8q 1V5\xda\x05\xed\xad\x7f`Q\x15\xc6\r\xea\xc7\xa2\x9cT\xfa\xc7\xd9()w#u#|\x93\x116\x14\xf0yLu\n\xf9\xcd\xd7R\x82?\xc8M\xd3G\xf3\xdaCy\xdc:e\x04\xfd\xe3b\xd0\x14\x85\xc4\x18;S/\x1bHK{\x80\xed\xfb\x8e\x81\xe0\x1b\x05E\xad4\rJ\x9c\x15\x08!\x04\x1f\xdc?\x9e)C\xe2\xbb\xb5rO_r\x9b+)\xc6\xc2\xb2\x99\x99\x1c^5\x9d\t\x83\xe3 \x14n\xf3*\nxN\x98\xc9\xa0\x1e\x0exRb\xb7j\xac{(\x8f?,\xbd\x9a\x9c\xf4\xa4\xe8Y]\xcf\xbcf\xc2\x10\x12R\xe2!\xd0Q\xa9\xa0h&\x90\xf0\x06g4\xc2\x90\xc1\xbf\xe7e\x91;\xd0\x15\xddT\x9c\x83!K)\x19\x9c`\xe7\x8b\x9f~\x9f\x9f\xe7g\x13g\x05\xea\x8f@Km\xd4\xee\xac\x9a\xc2\x15\xc3\x07\xf0\xf9\x13w\xb2d\xfe8,\x9a\xda\xe3\xac\xc8\xdf\xf4\x04\x97L\x8dsA\xc8\xbfK\xa3\xbd\xb9\x03G\xdc\x1e\xa41\xee\xcc\xadMHK\xb0\xb3\xfa\xaa\xacs\x06\xd0\xbd\xc3c\xde\xe6/\xf2\x861g\xb0\x9d\x17\x8a\xfcl\xaf\x80@\xc8`tz\x12\xb7^q\x19\x0f\xce\xbd\xbc\xdb\xcd\x9f\xee,\x87U\xc5*$\xa8\x96\x8f5\xbc\x8d>\xe2\x07}H\xd0?\x0e\x8b\xed\xf4v\x122\xb8}|\x1a\xaa"\xce\xa8N\xe7\x02f\xe6\xa8AL\xcaJ\xe5h\xa3\x07\xdd0\x18\x98\x10KB\xac\xfd[U6\xb2\xdauEH#\xd4\x82\xa1\xeeU\xd8\xfaj\x08\xbfg+\x81\xb63g[\x86d\xea\xd0\x84\xf3f\xa370\x00N\x9b\x85\x91\x83\x12\xc9\xc9H\x8e\x828\xcb3\xe4\t\x8f\x1f\x9f\xaf\xa3\x98\xc6\xea\x0e\xd3\xe76T\x96\x91\x90\xbe\x17)\xc7\x9c6\x80II\xb2\xd3\xd2\xe7.hDH\xdd\x90\x1c\xacu\xf3R\xc9Wl\xd8q\x08\xab\xaa\xf2\x93\x05\xe3\xb9gf\x0e\x89\xb1\xf6\xb3aD\x1c\xabk\xa0\xaa\xee\xe4\xfb\xecx\xc5g\x02\xd9\xfa\xda\x11R\x87\x97\xe0J\x19\x8d\xa2\x89^\x99\x11\x82\x86\xb6`\x9f@D\x04\xab\xa8;\xc9K%e\xfc\xae\xf8K\xda=\x1d\xa0\x99\xed\xb5\xe5/\x17\xb3i\xcf\x11\xd6\xdd;\x87\x91\x03\x13M\xe0B\xf4\x10FJI dp\xa4\xaei\x7fp\xfb\xe7\xa5\x00Jt\x90\xd9P\xf1\x9fx\x1a\xda\xa2>\xbcG\x1a \xf80\xdc\xdb\x95R\x9e\xb7:\xfdj\xe3\x0e\xf2\xd7\xbcK\xc1{;i\xef\x08\x98 "\x91\xddi\xa3d\xcf\x11\xae]\xfb.\xa5\x07\x8f\x9b\xc0Os\x96\xd7\x170\xde(\xdaQD\xf5\xdb\x87\xcdNc\xa4\xd7[\xb5\xbb\x81\x01\xa3\xd2I\xcc\x98\x8c\xa2\xc8h\x05\x16Y\x8a`\xd7\xb1VV\xce\x1e\x82US\xce\xca\x18\xbb~\xa6\xf4`\r\xf9\xbf\xd9\xc8\x86\x9d\x87h\xf2t\x80\x94\xb8\x9c6r\x87\xa4\x90\xd9?\x0e_ \x14\x05\xe6\xf6\xfa\xd8\xf2\xd5Q\xa6\x8d\x18Dz\x92\xab\xdb>\xe1\xe7\xf2\x95\xe2]-\xeb\xd7\xac_L\xfb\xc1F\xf2V\x9c\xc2\xda\xe5\xf3,\x0c\x9bYK\xc6\xe5Ih\xb6\x9e\x92\xea\x06\xf3F&\xf3\xee}\xe3p\xf62\xff8\x15\\ \xa4Ss\xb2\x8d_\xbe\xb3\x9dW\x8b\xf6\x80U\x03!\xc8\xee\x1f\xcf3w\xcfb\xe1\x84\xacn\xdf\x7ff\xd3.\xfe\xf5\x8f\xff\x8b?\xa4\x83n02#\x89w\x1e\xb9\x96\xd1\xe1\x80\x18I\x1b\xf7V70\xf6\xfe\'\xd7P\\\xf03\xf2V\xc2\x96\xb5]{\xbf+\xa0t\xbdA\xe6\xb8]\xf8=w!\x10\x18\xba\xc4\xd0\x05F\x08\xf4\x10\xc8\x10\x87k\xdc\x94\xec?N\xbaK%#\xc9\x85\xd6\xa5y\x1d\x01\xd1\xd0\xda\xce\xd6\x03\xc7y~\xcb\x1e\xee{\xa9\x88\xcf\x0f\x1c\x07\x9b\x05$\xdc~\xe5H6\xfd\xecFl\x16\x957\xff\xb6\x8f?\x94\x1e\xa0\xb8\xac\n\xb7\xd7\xc7=3G\xb3\xb7\xba\x89}\xc7\xdd\xa0\x08\x1a\x1b=\xfc\xad\xbc\x961\xe9Id\xa6\xf4\x03\xe0\xa3\x83\'\xc4\xddk\xff\xf8I\xd3\x9f\x7f~\x17\x00\xc3\xa7@\xc5\xf6S\x18\x89\xccF\x16\xaeZ\x8d\xc3\xf5+4\x9b\x99\xf7D\xab\xb3p7C\xd7M\xb5\x18\x9c\xcc\x98A\x89$\xba\x1ch\x9a\x8a\xdf\x1f\xa4\xb2\xbe\x85\xc3\r\xad\x94\xd7\x9d\xc4\xe3\xf1\x99\xe37! \xa43/w\x08E\xabo\xe6\xad\xd2\x03,_\xff)u\x8d\xad\x9d\x9ak\xd5x\xfb\xe1\x85\xe8Rr\xebs\x1ft\x89_:\xa9\x89\xb1\x8c\xcfL\x95zL\xb2\xd8{\xa0\xfcH\xdd\xde=3\xd8\xba\xeeX\xd71yw a\x9a\x98\xbb\xd4\x89#\xeeu\x9c\xf1\xb7D\x07\x95g0\xe2Nu\x92\x18\x86\xec=\xb35$)qN&g\xf5\xa7\xb0\xac\n]7:\xff\x1f6\xe8\xf4\xc4X\xfc\xbaNC\xab\xafg\x91\xd5/\r\xf4\x90\x87\xda\x03\xd7\xb2\xe97\x9f\x92\xb7\x02\xb6\x14\x9ca\x18\x1a\x19\xbbM\xbd\xd7E\xd2\xc0\xcd8\x13\xa6\x86=\x99\t\xe8l\x83\xa1\x94\x9d\x85\x90\xe8\x92\xe9\x1a\xf2\xf4\xb3\x94\x88\x87\x92\xd2L&#\x13_W\xaa@\xb5z8\xb2\xf3\xa7\x14\x16\xbc\xd2\xdbW{\xee\xb8y-\xe4\xad\x80\xd2\xd7<4\x1e\xcf\xa7\xad\xf9/\xe1\xacT\x84\xd99+\x10\x0e\xab\x85I\xc3\xd3\xb0j*\x04\xf5N\x17\xdb\x15DW\xc1u\x03\x02!\x92\\\x0e\xee\x9c>\n\x0cC\xa2j\x92\xb8\x01\x02\xc3h\xe1\xc8\x8eeQ\x10\xbd\xfc\xee\xab\xf7ah\xe56\xc8[\x0e\x9f\xbc\x10`p\xee\xfb\xe6\xcd\x8a\xe9\xa8\x96\xb3\xa3C\xc2\xfc\xdc!lxd!\xb3G\xa5S\xef\xf5QY{\x12:\xfc\xa6\xc0\xba\x0e!\xddL\xe5\x03A\xd0%\x99\x03\x13y`n.O\xdf5\x93E\x13\x87\xf1\xf4\xa62A|\x9a\xa0\xad\xe9\x10\xf5\xe5w\xb2\xb9`cd\x84@\xe1\xda^\x86\'gZ\x11c\x9a\xbfTA\xb1/\xc2\xee|\x1dG\\?\xa0g\x9c9\x85\x91\xac\xfe\xf1\xbc\xbfb\x119\xe9I\xf8\x02!Z;\x02\xec?\xee\xa6\xba\xa9\x15\xb7\xc7\x87aH\xfa\xc5X\xc9L\x89#;-\x9e\xfeqN\xec\x16\x8d\x17\x8bw\xcbU\x9b\xca\xc5I\xbf\x02\xf5\xe5\xef\xe3mZ\xcc\x96\xa7j\xba\xc9\xd3\xeb\x14\xe8\xdbV\xfe\n\x90\n\x14>\t\xb3\x7f\x9c\x8c3\xf65\x1cqW\xa1Y\x1d\xdd:~\xa7\x023\x0c\x84\x10\xac\xbcv\x12\x0f\xcc\x19K\x8a\xcb\x81USP\x15%j\xdf\xbaa\x10\xd4\r\xdcm~\xde\xff\xe20\xcf\x17\xef\xa5\xd2c\t\xe1on\xa2v\xdf\xe3\x14\xff\xf6w\xe1\xc1,l\xe9\xcb/\x1f\xba\xb1\xd3\x85\xd2\xf9\xcb\xe6a\x8fy\x00\x8bc\x06\x8a\x9a\x86\xaau\x02\xea\xda\x97\x95\x12\x02!\x90\x92!\xe9I\x0cK\x8d\'\xc6n&\x9e\x1d\x81\xa0lm\x0f\x88\x1ao\x90\xea\x9a\x16\xd3\xb8m\xf6]\xb4T\x7f\xc07\xbb\x9f\xe5\xcb\xbf\xb8\xcf\xa4J\xe7\x0f\x04\xe8\xe6\xf2.\x9b\xa90db.\xf6\x98\x19\x08\xe5\x1aTm\x0e\x8aEE\xb3t\x8f;]\xca\x80hW]\xb3\x85\xdf\x0b\x814\xaa\x11\xca\x16\x8c\xc0fj\x0el\xa7\xf4\xf5\xda\x1eg\x9d\xd5\x80\xf1|\xd6\xfceP\xf4\xb4\xf9|\xd6\x83\x02\xab\xd3\x81\xd5\x11\x83\x94\x93\x90\xc6,T\xcb8\x145\x1bEI\x02\xe1D\x08\x15)C\x08\xc5\x83P\xeb\x08u\xec#\x14\xdc\x89\xc5\xbe\x05\xa8\xa2\xb9\xa6\x83O^\n\xf4\xa5D\xf8\x7fM\xdd#\x1d\xf5i\xe0U\x00\x00\x00\x00IEND\xaeB`\x82 \ No newline at end of file From 77e22e9ff271517ceab707dca0c260ae6f9ac282 Mon Sep 17 00:00:00 2001 From: chris Date: Wed, 18 Jun 2014 16:08:53 -0400 Subject: [PATCH 32/82] Added test image. --- database_files/tests/fixtures/test_image.png | Bin 0 -> 3790 bytes 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 database_files/tests/fixtures/test_image.png diff --git a/database_files/tests/fixtures/test_image.png b/database_files/tests/fixtures/test_image.png new file mode 100644 index 0000000000000000000000000000000000000000..69914db46d35c441034d7b92c51ccf2d799a09a3 GIT binary patch literal 3790 zcmV;<4l(hGP)Px#24YJ`L;(K){{a7>y{D4^000SaNLh0L03N~s03N~tZqRi;00007bV*G`2iyo1 z4KM=74_*`i01i$`L_t(&-qo6UbW~NI$3OSIS8Gya4*?PaA;=OiEDbiYBM_n}?n~P` zZqv%pv^IXz4J0=~gMU`ZXpTP~kc$M^Ag~Ec10!F!j#!bxis{wEhNuM!m zY}V{yDgCf1ia@gp2ISn!6}PS7${#vRFw430R@WQOzGMFCH;YdgQpyj@-G*ybiDcOp zgXKk%CGSXXd2%D)yCuEzyB7wK+UFheS3qHb`F!S2R;_<}$+JJc=KD$e8`|ZsOTH0b z*Ljf$0Z25Pz?D1_SMnVs4{hei&fQdOH!q@MTc5-b=gmMw_@TEd)|{*XxkCrM_}t<} z3v;s4DL&+8%fTjI-+7FnZeZdFFDWHL2s&EfMEMQ^-qlF9akAP^>F!*h&?rJLIV5i+wl5B-GPWOI~KbjeqcIQ=iMmkEyN_+ z$;fS?Vp~J>vd)1eaV{$4RBg2X^$B1~@@4nm@bKbmr`$em^pNcD4NnJ>~42vM}d%@RQgS&bIbqCh6ZT&HR@LO;n?PR~tK=N0bF#urF zkM3J?Z~haH-8?I0K%z_4R5c0$;JvRKsc7op#nPixG_;xTiZdJ&1%yCKFm#E#x}M`@ zk5RJj4fFivu=%Mz0O<-~&cn$=GY7o+#II0iQcA3LaJX{NtW$|i z`jn4faG%ZzsJ+I8&0*u3vGnnxwM%|^^R%(CHj5_1Ob~fgH~ZLhz)el7pa1#l1Rs?* zb-&d<1qNZ-jA{XAd{mON$DYJU(xHwuOkvhCOG< zfL+JEd|cU#$EWl1_G&(_Y6eAI7!ciEwE!V;#`VYQ98E&{2YmD&zEi=dJpx_DHXMsq z&3)#f>tDQW`sFbZ073|KLsI7n@JVGeyKB5uHnsEo)+(yq9T!HO&ICwNL9=Hgbx6r9 z!RqQ}FKRctC@erMSe~?K`o#BE-!^-I)27LYQP;Hi`M+Zxc2;|+Xlkc$e*+u8Izf9- z2jwDy=wdZkYyw})G(w?06s|qMf|XGX(z<>Q!0C#6{kl0*h9$(ne$Q3^l1bs(laX|(yF$-ceEqEJZrb>R={ZtU zr0575gbY*I!%gkH^+gR0-oS;mgL8>5iV96t5?`~RrT!f@t-0AWiIpf_(rAse;RViw z6#T)EXle^m?+x&Et(SFMt9b6C!!&sQP9UP5P=Yi}<9Ea_VE!XhO^^l}BCOw6KXxDe zvoby@Zk5oYBotPZC_zy3B<>nv0iB;t}}0|DMfQ>`MZ9$`hZ zSdns~f~qB9u_yJRg1`7|_`Tq5AnIYe21(Td(wL#32S;Ix0eOk3l?;oc2f$s`8;Ecb zrh=m8Di}s47CRutC21G#>aGUfdl9B+Sqf4nqgpRPPh~)J4FuZ1HJ3zDk~ENNpJRoT zJ@)QqZ|Aaqq-H-^4ZfCeIU;)BH=4t14(+@x+>j`$U~qz+8%C!xcSIuD$xafT7BofS zP(vGk{konFWet3DvV#tvZl*e?Zw+-a;XVo-p3d?hd{Jo$G4!+BXb%KA(b7&ZsGETs zMQRAV=pj(Gvj?FYB)e=ZotDAksTrii^i<8sgX5SyIF42GhViG}wYu9O?m ztn#cJu#^l+aIo>7Tz)ulATd@o+6C>ce_&A+#trJnqMTIv$Jr<;cO!Kpe3KD26!3XW zbqe{-*HVH%$W7Bn^V&ms+&X0h{o`WL6oteXJ5w(kNK1Qw?fZ^_;2Q-+tuzfvoD~Qm z1RvZ#o+(4(yR~vpUvir(DaOLGnVB43HJu5A;~*4n^C}8>PMFD{9u9a?Vz+9ndnk`T zEWMhX^rYxnUC+E`!VnTt;vvvcsi0^kknjd)G{TUW@H6ZPf0pn!**s2k`k5cd}&uI4qjlV6HnQ4vMMxvmf}!qJ*!pS+zh^6fwlim2hGgo{g!5*jH!uKBn}v~s63NL( zj)FMLGvo=6AD^$In8xZ3)8{r+mXVOY7b(Y_Hh@V#MYpsMcV`|NndX9C&!+7VKRm*7enjiQNOnGu49wqTz? zLp0_&vYU2*_*St%?dpl|G2$&!uk=<#dWX}<< zUHdMj2alV;Pji;n7ckx^+EGOH+lS3gW2DzxwYz}l$_${4$Tru1M9`M=r7hMk9K@^pZdxSTd}hoq3%KMr3Y6b0t?`q=W-QKpR;M0%2| z+dkpv@|EIUE$i2;UrhVKM)Fpi!rJBYEDW1jOUC6%Xtth`>IU;hCiCun<8bye|2P#~ zAf!`svW;K9vz=Fq_JCCbA;|6@&og(=V&R0WZr^`q(=Pt?kN=B5q=RlSG9!t19=Vp$ z;eZ%P8~0W>F!uf@*HB#WGxAo#mbG1bzbl}0y+P)cpJ^Zb_mNDr`WTx~3;G=^l#NBKE_({{Be|{GL z!>2&S_EQ*%%tjWjT8+c?D>R##_e86+a5qEOO|=-D%$PxmB)S}Enu^~aq_Vz+!ws#J z*PX=U@tNZ{At0nPFLwwvJD;iC;RYiLs2& zOqF^}ve4XUvo$0s zaZV;@^{23`3SBok{~tE!^u$>Fx=ur@uV<0fFAemNhuQ8uIkBl!%9t(o7~J6JJ-VK(%@}vh$}D7QE8i zS9|Wbc`f9vfYR4IG#<~Vt@$tEtW<>AIV%tjtA$C!)39nP1ociA+Z9At!F2-yA(C7U z?wmdf3`3%7lDGi^!)W2ij%860z5MR0_hD$|Hpp8JpS%!YaPIr&&5G%$mb0Tn!u;GJ zYB;n13}OwnFAAfU~Vo?-!+p(6NmBirZPc%nxL)e z5cTDEvbkVmWQ0KB+FphyXIc?sn!i**u`j~zd>uzzKcHuvIfap|{_(uOVi7s%N%#UG zS~~*lKklZwsTH@!hha$i#aPKqi6c8Lp8j!8>=um|i+9Uao65yWzXJ8;@8fM+%$BEW zy2;b)6zJPl{tA!^6g~;Ff1Hdn_B9-FSD{%Q-8z1!0y7MR5Ujj*5)aH7ONz^ZRZ~zE zC2HTgVGz^}+--i||Kc#u7w@IgV+rAJZlZSI;}rk)cj3V-Z0XA{AG)#Ah0^)U=3$R{ z0E=S=ikgP1bpqY5hQxNY`bv{Zk< zH@lzZ%fGnKq)O)x1nB1SE}N+^WMVG%m>CG=8dPl#ibX}UbdNh-%7AEBU9*MX3xPDM z5y}>fz-DR=Y^U_~+8$?})qwGQ*8F8q^tAbW)`J48(}C3ygOrny#w=9JI21J-MM(nT zL?0K1iN(GJsG_;1gHK)x^rb}je@)#Z9rbD8RR91007*qoM6N<$ Ef=4Pu8UO$Q literal 0 HcmV?d00001 From 7def8c9750f7287fcf135ade277340ea5c9d3ba0 Mon Sep 17 00:00:00 2001 From: chris Date: Wed, 18 Jun 2014 19:31:49 -0400 Subject: [PATCH 33/82] Added support for Python3. --- .gitignore | 59 ++++++++++++- README.md | 25 +++--- .../commands/database_files_cleanup.py | 18 ++-- .../commands/database_files_load.py | 13 +-- .../commands/database_files_rehash.py | 4 +- database_files/models.py | 22 +++-- database_files/settings.py | 31 +++++-- database_files/storage.py | 15 ++-- database_files/tests/settings.py | 16 +++- database_files/tests/tests.py | 82 ++++++++---------- database_files/urls.py | 7 +- database_files/utils.py | 60 ++++++------- database_files/views.py | 2 +- setup.py | 86 +++++++++++++++---- 14 files changed, 291 insertions(+), 149 deletions(-) diff --git a/.gitignore b/.gitignore index 9a2ec7c..c619fee 100644 --- a/.gitignore +++ b/.gitignore @@ -1,6 +1,59 @@ -*.pyc -/build + +# Byte-compiled / optimized / DLL files +__pycache__/ +*.py[cod] + +# C extensions +*.so + +# Distribution / packaging +bin/ +build/ +develop-eggs/ +dist/ +eggs/ +lib/ +lib64/ +parts/ +sdist/ +var/ +*.egg-info/ +.installed.cfg +*.egg +MANIFEST + +# Installer logs +pip-log.txt +pip-delete-this-directory.txt + +# Unit test / coverage reports +.tox/ +.coverage +.cache +nosetests.xml +coverage.xml + +# Translations +*.mo + +# Mr Developer +.mr.developer.cfg .project .pydevproject .settings -/dist + +# Rope +.ropeproject + +# Django stuff: +*.log +*.pot + +# Sphinx documentation +docs/_build/ + +# Local virtualenvs used for testing. +/.env* + +# PIP install version files. +/=* diff --git a/README.md b/README.md index a5754de..922d61e 100644 --- a/README.md +++ b/README.md @@ -9,51 +9,52 @@ but there are some valid use cases. If your Django app is behind a caching reverse proxy and you need to scale your application servers, it may be simpler to store files in the database. -Requires: - - * Django 1.1 - Installation ------------ - sudo python setup.py install + python setup.py install Or via pip with: - sudo pip install https://github.com/chrisspen/django-database-files/zipball/master + pip install django-database-files You can run unittests with: python setup.py test +You can run unittests for a specific Python version using the `pv` parameter +like: + + python setup.py test --pv=3 + Usage ----- -In ``settings.py``, add ``database_files`` to your ``INSTALLED_APPS`` and add +In `settings.py`, add `database_files` to your `INSTALLED_APPS` and add this line: DEFAULT_FILE_STORAGE = 'database_files.storage.DatabaseStorage' -Note, the ``upload_to`` parameter is still used to synchronize the files stored +Note, the `upload_to` parameter is still used to synchronize the files stored in the database with those on the file system, so new and existing fields should still have a value that makes sense from your base media directory. If you're using South, the initial model migrations will scan through all -existing models for ``FileFields`` or ``ImageFields`` and will automatically +existing models for `FileFields` or `ImageFields` and will automatically load them into the database. If for any reason you want to re-run this bulk import task, run: - $ python manage.py database_files_load + python manage.py database_files_load Additionally, if you want to export all files in the database back to the file system, run: - $ python manage.py database_files_dump + python manage.py database_files_dump Note, that when a field referencing a file is cleared, the corresponding file in the database and on the file system will not be automatically deleted. To delete all files in the database and file system not referenced by any model fields, run: - $ python manage.py database_files_cleanup + python manage.py database_files_cleanup diff --git a/database_files/management/commands/database_files_cleanup.py b/database_files/management/commands/database_files_cleanup.py index daa2138..af5f50e 100644 --- a/database_files/management/commands/database_files_cleanup.py +++ b/database_files/management/commands/database_files_cleanup.py @@ -1,3 +1,5 @@ +from __future__ import print_function + import os from optparse import make_option @@ -28,7 +30,7 @@ def handle(self, *args, **options): dryrun = options['dryrun'] try: for model in get_models(): - print 'Checking model %s...' % (model,) + print('Checking model %s...' % (model,)) for field in model._meta.fields: if not isinstance(field, (FileField, ImageField)): continue @@ -41,7 +43,7 @@ def handle(self, *args, **options): for row in subq.iterator(): subq_i += 1 if subq_i == 1 or not subq_i % 100: - print '%i of %i' % (subq_i, subq_total) + print('%i of %i' % (subq_i, subq_total)) file = getattr(row, field.name) if file is None: continue @@ -49,22 +51,22 @@ def handle(self, *args, **options): continue names.add(file.name) # Find all database files with names not in our list. - print 'Finding orphaned files...' + print('Finding orphaned files...') orphan_files = File.objects.exclude(name__in=names).only('name', 'size') total_bytes = 0 orphan_total = orphan_files.count() orphan_i = 0 - print 'Deleting %i orphaned files...' % (orphan_total,) + print('Deleting %i orphaned files...' % (orphan_total,)) for f in orphan_files.iterator(): orphan_i += 1 if orphan_i == 1 or not orphan_i % 100: - print '%i of %i' % (orphan_i, orphan_total) + print('%i of %i' % (orphan_i, orphan_total)) total_bytes += f.size if dryrun: - print 'File %s is orphaned.' % (f.name,) + print('File %s is orphaned.' % (f.name,)) else: - print 'Deleting orphan file %s...' % (f.name,) + print('Deleting orphan file %s...' % (f.name,)) default_storage.delete(f.name) - print '%i total bytes in orphan files.' % total_bytes + print('%i total bytes in orphan files.' % total_bytes) finally: settings.DEBUG = tmp_debug diff --git a/database_files/management/commands/database_files_load.py b/database_files/management/commands/database_files_load.py index 6139c16..b28b5a6 100644 --- a/database_files/management/commands/database_files_load.py +++ b/database_files/management/commands/database_files_load.py @@ -1,3 +1,5 @@ +from __future__ import print_function + import os from django.conf import settings @@ -36,7 +38,8 @@ def handle(self, *args, **options): for field in model._meta.fields: if not isinstance(field, (FileField, ImageField)): continue - print model.__name__, field.name + if show_files: + print(model.__name__, field.name) # Ignore records with null or empty string values. q = {'%s__isnull'%field.name:False} xq = {field.name:''} @@ -48,10 +51,10 @@ def handle(self, *args, **options): if not file.name: continue if show_files: - print "\t",file.name + print("\t",file.name) if file.path and not os.path.isfile(file.path): if show_files: - print "Broken:",file.name + print("Broken:",file.name) broken += 1 continue file.read() @@ -59,7 +62,7 @@ def handle(self, *args, **options): except IOError: broken += 1 if show_files: - print '-'*80 - print '%i broken' % (broken,) + print('-'*80) + print('%i broken' % (broken,)) finally: settings.DEBUG = tmp_debug diff --git a/database_files/management/commands/database_files_rehash.py b/database_files/management/commands/database_files_rehash.py index ad11b21..ab23bb0 100644 --- a/database_files/management/commands/database_files_rehash.py +++ b/database_files/management/commands/database_files_rehash.py @@ -1,3 +1,5 @@ +from __future__ import print_function + from optparse import make_option from django.conf import settings @@ -28,7 +30,7 @@ def handle(self, *args, **options): total = q.count() i = 1 for f in q.iterator(): - print '%i of %i: %s' % (i, total, f.name) + print('%i of %i: %s' % (i, total, f.name)) f._content_hash = None f.save() i += 1 diff --git a/database_files/models.py b/database_files/models.py index 2445cfa..c833072 100644 --- a/database_files/models.py +++ b/database_files/models.py @@ -1,6 +1,10 @@ +from __future__ import print_function + import base64 -import settings as _settings +import six + +from . import settings as _settings from django.conf import settings from django.db import models @@ -57,7 +61,10 @@ def save(self, *args, **kwargs): @property def content(self): - return base64.b64decode(self._content) + c = self._content + if not isinstance(c, six.binary_type): + c = c.encode('utf-8') + return base64.b64decode(c) @content.setter def content(self, v): @@ -94,19 +101,20 @@ def dump_files(cls, debug=True, verbose=False): tmp_debug = settings.DEBUG settings.DEBUG = False try: - q = cls.objects.only('id', 'name', '_content_hash').values_list('id', 'name', '_content_hash') + q = cls.objects.only('id', 'name', '_content_hash')\ + .values_list('id', 'name', '_content_hash') total = q.count() if verbose: - print 'Checking %i total files...' % (total,) + print('Checking %i total files...' % (total,)) i = 0 for (file_id, name, content_hash) in q.iterator(): i += 1 if verbose and not i % 100: - print '%i of %i' % (i, total) + print('%i of %i' % (i, total)) if not is_fresh(name=name, content_hash=content_hash): if verbose: - print 'File %i-%s is stale. Writing to local file system...' \ - % (file_id, name) + print(('File %i-%s is stale. Writing to local file ' + 'system...') % (file_id, name)) file = File.objects.get(id=file_id) write_file( file.name, diff --git a/database_files/settings.py b/database_files/settings.py index 5c0fa1b..58042ee 100644 --- a/database_files/settings.py +++ b/database_files/settings.py @@ -3,6 +3,13 @@ from django.conf import settings from django.core.urlresolvers import reverse +# If true, when file objects are created, they will be automatically copied +# to the local file system for faster serving. +settings.DB_FILES_AUTO_EXPORT_DB_TO_FS = getattr( + settings, + 'DB_FILES_AUTO_EXPORT_DB_TO_FS', + True) + def URL_METHOD_1(name): """ Construct file URL based on media URL. @@ -20,17 +27,27 @@ def URL_METHOD_2(name): ('URL_METHOD_2', URL_METHOD_2), ) -DATABASE_FILES_URL_METHOD_NAME = getattr( +settings.DATABASE_FILES_URL_METHOD_NAME = getattr( settings, 'DATABASE_FILES_URL_METHOD', 'URL_METHOD_1') -if callable(DATABASE_FILES_URL_METHOD_NAME): - method = DATABASE_FILES_URL_METHOD_NAME +if callable(settings.DATABASE_FILES_URL_METHOD_NAME): + method = settings.DATABASE_FILES_URL_METHOD_NAME else: - method = dict(URL_METHODS)[DATABASE_FILES_URL_METHOD_NAME] + method = dict(URL_METHODS)[settings.DATABASE_FILES_URL_METHOD_NAME] +settings.DATABASE_FILES_URL_METHOD = method + +settings.DB_FILES_DEFAULT_ENFORCE_ENCODING = getattr( + settings, 'DB_FILES_DEFAULT_ENFORCE_ENCODING', True) -DATABASE_FILES_URL_METHOD = setattr( +settings.DB_FILES_DEFAULT_ENCODING = getattr( settings, - 'DATABASE_FILES_URL_METHOD', - method) + 'DB_FILES_DEFAULT_ENCODING', + 'ascii') + +settings.DB_FILES_DEFAULT_ERROR_METHOD = getattr( + settings, 'DB_FILES_DEFAULT_ERROR_METHOD', 'ignore') + +settings.DB_FILES_DEFAULT_HASH_FN_TEMPLATE = getattr( + settings, 'DB_FILES_DEFAULT_HASH_FN_TEMPLATE', '%s.hash') diff --git a/database_files/storage.py b/database_files/storage.py index 56a06c1..48a97b1 100644 --- a/database_files/storage.py +++ b/database_files/storage.py @@ -1,5 +1,8 @@ +from __future__ import print_function import os -import StringIO + +import six +from six import StringIO from django.conf import settings from django.core import files @@ -9,8 +12,6 @@ from database_files import models from database_files import utils -AUTO_EXPORT_DB_TO_FS = getattr(settings, 'DB_FILES_AUTO_EXPORT_DB_TO_FS', True) - class DatabaseStorage(FileSystemStorage): def _generate_name(self, name, pk): @@ -31,7 +32,7 @@ def _open(self, name, mode='rb'): f = models.File.objects.get_from_name(name) content = f.content size = f.size - if AUTO_EXPORT_DB_TO_FS \ + if settings.DB_FILES_AUTO_EXPORT_DB_TO_FS \ and not utils.is_fresh(f.name, f.content_hash): # Automatically write the file to the filesystem # if it's missing and exists in the database. @@ -45,6 +46,7 @@ def _open(self, name, mode='rb'): # and load it into the database if present. fqfn = self.path(name) if os.path.isfile(fqfn): + #print('Loading file into database.') self._save(name, open(fqfn, mode)) fh = super(DatabaseStorage, self)._open(name, mode) content = fh.read() @@ -53,7 +55,8 @@ def _open(self, name, mode='rb'): # Otherwise we don't know where the file is. return # Normalize the content to a new file object. - fh = StringIO.StringIO(content) + #fh = StringIO(content) + fh = six.BytesIO(content) fh.name = name fh.mode = mode fh.size = size @@ -77,7 +80,7 @@ def _save(self, name, content): name=name, ) # Automatically write the change to the local file system. - if getattr(settings, 'DATABASE_FILES_FS_AUTO_WRITE', True): + if settings.DB_FILES_AUTO_EXPORT_DB_TO_FS: utils.write_file(name, content, overwrite=True) #TODO:add callback to handle custom save behavior? return self._generate_name(name, f.pk) diff --git a/database_files/tests/settings.py b/database_files/tests/settings.py index 4b1447f..53d5758 100644 --- a/database_files/tests/settings.py +++ b/database_files/tests/settings.py @@ -1,5 +1,7 @@ import os, sys + PROJECT_DIR = os.path.dirname(__file__) + DATABASES = { 'default':{ 'ENGINE': 'django.db.backends.sqlite3', @@ -8,7 +10,9 @@ # 'TEST_NAME': '/tmp/database_files.db', } } + ROOT_URLCONF = 'database_files.urls' + INSTALLED_APPS = [ 'django.contrib.auth', 'django.contrib.contenttypes', @@ -16,6 +20,16 @@ 'django.contrib.sites', 'database_files', 'database_files.tests', + 'south', ] + DEFAULT_FILE_STORAGE = 'database_files.storage.DatabaseStorage' -MEDIA_ROOT = os.path.join(PROJECT_DIR, 'media') \ No newline at end of file + +MEDIA_ROOT = os.path.join(PROJECT_DIR, 'media') + +# Run our South migrations during unittesting. +SOUTH_TESTS_MIGRATE = True + +USE_TZ = True + +SECRET_KEY = 'secret' diff --git a/database_files/tests/tests.py b/database_files/tests/tests.py index 45871cd..c2dabaa 100644 --- a/database_files/tests/tests.py +++ b/database_files/tests/tests.py @@ -1,6 +1,9 @@ # -*- coding: utf-8 -*- import os -import StringIO +import shutil + +import six +from six import StringIO from django.core import files from django.test import TestCase @@ -14,14 +17,17 @@ class DatabaseFilesTestCase(TestCase): + def setUp(self): + self.media_dir = os.path.join(DIR, 'media/i/special') + if os.path.isdir(self.media_dir): + shutil.rmtree(self.media_dir) + os.makedirs(self.media_dir) + def test_adding_file(self): # Create default thing storing reference to file # in the local media directory. - media_dir = os.path.join(DIR,'media/i/special') - if not os.path.isdir(media_dir): - os.makedirs(media_dir) - test_fqfn = os.path.join(media_dir,'test.txt') + test_fqfn = os.path.join(self.media_dir, 'test.txt') open(test_fqfn,'w').write('hello there') o1 = o = Thing() test_fn = 'i/special/test.txt' @@ -42,21 +48,21 @@ def test_adding_file(self): # Verify we can read the contents of thing. o = Thing.objects.get(id=id) - self.assertEqual(o.upload.read(), "hello there") + self.assertEqual(o.upload.read(), b"hello there") # Verify that by attempting to read the file, we've automatically # loaded it into the database. File.objects.update() q = File.objects.all() self.assertEqual(q.count(), 1) - self.assertEqual(q[0].content, "hello there") + self.assertEqual(q[0].content, b"hello there") # Load a dynamically created file outside /media. test_file = files.temp.NamedTemporaryFile( suffix='.txt', dir=files.temp.gettempdir() ) - test_file.write('1234567890') + test_file.write(b'1234567890') test_file.seek(0) t = Thing.objects.create( upload=files.File(test_file), @@ -65,7 +71,7 @@ def test_adding_file(self): t = Thing.objects.get(pk=t.pk) self.assertEqual(t.upload.file.size, 10) self.assertEqual(t.upload.file.name[-4:], '.txt') - self.assertEqual(t.upload.file.read(), '1234567890') + self.assertEqual(t.upload.file.read(), b'1234567890') t.upload.delete() self.assertEqual(File.objects.count(), 1) @@ -87,53 +93,41 @@ def test_adding_file(self): self.assertEqual(os.path.isfile(test_fqfn), False) def test_hash(self): - verbose = 0 + verbose = 1 # Create test file. - media_dir = os.path.join(DIR, 'media/i/special') - if not os.path.isdir(media_dir): - os.makedirs(media_dir) - image_content = '\x89PNG\r\n\x1a\n\x00\x00\x00\rIHDR\x00\x00\x002\x00\x00\x009\x08\x06\x00\x00\x00t\xf8xr\x00\x00\x00\x01sRGB\x00\xae\xce\x1c\xe9\x00\x00\x00\x06bKGD\x00\xff\x00\xff\x00\xff\xa0\xbd\xa7\x93\x00\x00\x00\tpHYs\x00\x00\x1e\xc2\x00\x00\x1e\xc2\x01n\xd0u>\x00\x00\x00\x07tIME\x07\xdc\x08\x15\r0\x02\xc7\x0f^\x14\x00\x00\x0eNIDATh\xde\xd5\x9aytTU\x9e\xc7?\xf7\xbdWkRd\x0f\x01\x12\x02!\xc8\x120,\r6\xb2#\x10\xa2(\xeeK\xdb:n\xd3\xca\xd0\xb46\x8b\xf6\xc09=\xd3v\xdb\xa7\x91\xb8\xb5L\xab\xe3\xb8\xb5\x8c\xbd\x88\x83\xda-\x84\xc4\xb84\x13\x10F\x01\r\xb2%\x01C \tY\x8a$UIj{\xef\xce\x1f\xaf\xaa\x92\x90\x80@\xc0\xa6\xef9uRU\xa9\xba\xf7\xf7\xbd\xdf\xdf\xfe+\xb8Xk\xc1\x8aD\xe6/K\x03 \x7f%\x17{)\x17kcUU/s%\xa6\xbce[\xf8h6\x9b\xd7B\xfe\x8a\x7fL \xe9)I\t\xd3\'O\x9c-\xe2\x07\xfc\x89\x9c\xd9N6\x17\\Tf.\x1a\x90\xe1\x19\xa9\xa3\xe6\x8d\x19\x84\x96>f\x02#\xe6l\x02`\xf3ZX\xb0\xe2\x1f\x08\xc8\x84\xfb\xfb\xa5\xc6\xc7.\x9c;&\x1d\xabE\x95d\x8c\x9f\xc1\r\xbfx\x95\xf4\\\x85\xc2\x82\x8b\x02F\xb9\x80\xc6M\xcc\xf5\xab\x01p\x0eI\x9f1kl\xd6\xeca)\xfd\xb0)\x8a@\xb3\n\x06\xe4\xdc\xcb\x15\xb7\xad\xe1\xca\x1f:L0\xcb\xb9\xf4V\xd7\x1b\xce\xbec\xfe\xea7\x8bO\x18R\xca\x0f\xcb\xdd\x86kU\x89d\xd9\x16\x83\xe5E\x92%\xefHnyr#\xdf\xbb%\xbd\xf3\xbb\x17\x06\x90\xda\xe7\x1d\xf2W@a\x81\xf9|\xe6OV\xad\x7fz\xc9\xb3?\xbe\xe6\xfbI\xfb\x1b\xda\xe5\xafK\xbe\x11_\xd7y\x91\x08\x01H4\x9b\xc0\x95<\x12W\xf2\x1d$\x0fm\xe6\xc8\xce\xddTl7/\xa2b[\x9f\xc4\x10\xe7\xcd@D\xf8\xa1\xb7*\xd6\x9c\xac\x01\xb9C\x07\xbc\xf8\xdc\xe2E\x0bs\xb2\xd2)>\xe4f\xcb\xc1&^\xdfy\x8c\xa0n`\xe2\x08/)%B\x08:Z\xe1D\xf9\x0e\x02\xde\xd5H\xb6q\xb2\xdaO\xe9\xef\x8d\xef\x96\x91\xc8\xed\r\xbfcX\xde\xeds\xef\xbf\xe1\xca\xb1\x7f~n\xf1\xf5cGe\xa4\xcaV\x7f\x88=5mB\x02\xfb\xeb\xdb\xf0\xfaC\xdd\x81\x880;\x16\xbb ~`:V\xe7\xdd(\xea-\xc4$\xda\xc9\xc8m\xa1b[C\xf4\xb2\xce\x81%q.*\xe4Tm\xb4\xff\xf5\t`J\xf2e\xf7\xdf\xf0\xf0\xe2k\xa7\xdc>m\xf4\x90\xec\xef\rO\x07\x90\x80\x08\xea\x06\xaf}QGY\x8d\x87\xff\xfa\xec\x18\xbe\xa0\xde\x1dH\'3 \x84\x0c\x8b \xf0{\xc1\xd7Z\x83\xbbz\x03u\x07\xd6\xb2m\xfd1~\xf0[p\x1f\xedd\xbf\xcf@\xf2W\x9a1\x00`\xd2\x8f\xee[ro\xfe\x13\x8f\xdd<+e@\xa2K\xd5T5\n\x02\xe0\xbd\xaf\x1b\xa9h\xea\xe0\xc5\xd2\xa3T4\xb4\x99\xef\x8a3\x1c\x13\x05\x84@J0t\x89\xbb\xba\x9e\xe3e\x8fQ\xb2\xee\r\xf3\xfc\xe5\xb0\xf9\xa9>\x00\xe9\n`\xce\xc3\xc9C3\x07\xbc\xf9\xc6\xb2[\x17L\x1f3T\x86\xd5]tj\x0b|\\\xd9\xcc\xd7\'\xdax\xb7\xac\x8e\x92\xfd\x8d\xa0\x9d\x87w7\x81A[3\xd4\xee[G[\xd3c|\xf8|;\xf9\x8f\xc2\xe6\'\xcf\xc3F\xba\x80p,Z5\xfb\x9f\x17^\xf9\xde\x7f?z\xc7\xe4Q\x83S\x91R\n\x11\x96^\x08AP\x97|v\xb4\x95/k\xbd|x\xa8\x91\xc2\xb2z\xb0\x9c\xa7C\x8c\xb0gs\x803~20\x82\xd4\xecR\x8a\x9f\xf5\x90\xff(T\x94\x9e\x03\x90\xaeL,X\xb1d\xdd\x92\x1b^\\~\xe3\x8cA\t\xb1\x0e\xd9\x95E!\x04\xed\x01\x9d\x8f\x0f7SV\xd7\xc6\xa6\x03\rl\xda\xd7\x00\xea\x05\x88\xb3RJ\xacvp\xb8r\xd0\xacS\x89M\xfa\x94\x8f_p\x9f\xce\t\xa8\xbd\xc6\x85\xcda\xe3\x9a\xb1\xf4\xf1\xa2\xb5K~y\xf3\xb4\xb1\xb16\x8b&\xc3L\x10yT7\xfb\xd8t\xc0MM\xab\x9f\xff\xf9\xea\x04\x1f\x957u\xbf\xd5>\x05\x06a\xda\x8cj\x01g|\x06\xd6\x98\xd98b?\xe0\xd3\x97[X\xb0\x1c*\xb6\x7f\x0b\x90\x08\xda\xbc\xe5K\x8b\n\x96<>o|\xb6\x85\x88\xc9\x86=g\xc8\x80]\xc7\xbd|\\\xd9\x8c\xc7\xa7\xf3\xf2\xf6j>\xafj\x06E\\\x18\x10\xdd]\xb5\x00!q\xc6\xf5\xc7\xea\x1cMB\xfa\x07|\xf4\x1f\xbeS\xc1\xa8=\x02]\xc56\x1c\x8bV\xcd}\xe6\xa1\xeb_\xbcm\xfa\xe51\x11\x00B\x08tCR\xe7\t\xf0Ie3\xbbk\xbcT6\xb5\xf3\xfc\xd6*\xaa\xdd\x1d\x17F\x9d\xce\x04HQA\xb3g#uH\xc9,\xc1\xea\xeaf/j7\xbb(,@,X\x9e\xb4h\xfa\xc4\xf7V\xdf6{\x80\xd3\xa6\xc9\x88Q\xd7\xb4\xf8\xf9\xbfc\x1evV{\xa8hj\xa7p\x7f\x03\x1b\xbf:A{P\x07\xe5"\x82\xe8b5Xl\x02_\xcb4B\xa1=\x14\xae=\xc0\x82\x95Q\r\xd2\xba~\x0e\xc0\xe9\x8a{\xfd\xba9S\x86%\xc4\xd8$ \xaaN\xfa\xf8\xaa\xd6K\xbd7H\xc8\x90\x94\x947Rt\xb0\x89\x96\x8eP\xf4;\xdf\xd1\x12\x08Eb\x8b\x15\xd8b~\xcfU\xff\x92I\xe1\xda\x93\x91t\xa9\xbbB\xcfX|\xfd\xac\xebn\xda\xf8\x83\xe99RS\xa4\xe8\x08\x1a\x84\x0cS\xd8\xc3M\xed\xbc\xf5E\r\r\xde\xc0\x85\xb5\x83s\x89/\x8a\nMU\x12_\xb3\xa0\xa5\xfe\x1d6\xad\xb9\xd94\x89\x95(]\xd2h\x8d\xb4\xe1\x05\xce\x84\x14\xfc\xc1\x90hj\x0bR\xef\r\xf0u\xad\x97u[\xabx\xee\xa3\xc34x\xfeN \xa2\x9eP\x824L\xe3w\xc4/`\xfe#SM\x90\x06\x1a"\xac\xdfW?v\x1f\xfd\xb32>)o\x94\r\x1e\x9f\xf0\xeb\x065->\x1a[\xfcf\xf9\xa5*\x7f?\x10\x91e\x18 u\x90R`\xb1\xc4`s=H\xca\x88mly\xcaL1\x18\xb70\x86\xb1W\xbfM\xf2\xd0|\x84\xd2\xa9\xfa\x82\x0b\x17\x17.\xc4\n\x05\xa0\xe9\x1b\x08\xf9\xcd\xd7A\xdfQ<\x8d\x0b)z\xa6\xcc\xa4\xa3\xff\x8818\xe2\'\xa1\xa8\xa6\xd0\x8a\xe8\x8c\t\x97\n\x08\x80\x90\x0f\x8c`\xe7\xe5*\xea`l\xb1S\x99\xf6O(dM\x82\xf8\x81\x13\xd0\xac\xc9\\\xaaKJ\xf3\xe1o\x03#Z\x16HT\x0b(\xcau8\x13\x1d\nC&\xd9\xd1,Wa\x8b\xe5\xbb\xf6\xa7\xe7d\xe8z\x10\xda\x9b;\xf5]J\x11f\xe5*\xac\x0e\xa7B\xbf\xfe6T\xeb,T\x15$\xe2\x92d\x03\x01\xdeFS\xad"\xaa\x1eaE\xb3X\x91r\xa2\x82\xaa%a\xb1\'\xf5\xa1\x82\xbf\xf8l\xf8\xbd\xe0m \xeaa\xbb\x06I\xd5\x02\xd2\x98\xa1\xa0\x07Ga\x8b\x01y\x89\xa9\x95\x0c\x8b\xa3\x07\xc0]\xd5\x1b\x88p\x11\xa6\x82\xa2\xe6*\x18F&\x16; \xc5%i\x17\xee\xea\xae\x06\xde{\x11\xa6hY\nR&\xa3Z/AOe@s\r\x04\xda\xbf5\x97DQ\x924\x90j\x9fc\x85\x94=\xf6\xeefo\xe7\xb2\x7f\xa4f?Y\r\xbe\x96p9"z\xdf7\x9a\xc3k\x0e\xed\xbc-\xdc\x90(\xaa`p\x82\x9d\x1bF\xa72wD"\xd9\xc9N\x12\x9d\x164E\xe1PC\x1b\x7f\xfd\xba\x9e\re\r|s\xb2\x83\x0e\x9fn\xa6:\xa7o\ru2\xe1>\n\x1d\x9e\xce\xf2 |Qi\t1\xf4\xb3\xdbh\x0f\x049\xd1\xd2N0\xa8\x9b\x81\x1bEj\x08\xbc\xe8!P\xb5\xb3\x07\xa1\x1b$\xbbl,\x9d\x96\xc1\xe2\xa9\x19\xa4\xc4\xf4T\xcd\xc9\x83\xe3\x98<8\x8eU\xf3\x86\xf1\xa7\xdd\xb5\xbc\xbc\xa3\x86\xd2\n\xb7Y\x01\x9d\n&\xf2:\xe8\x83\xe6\xe3\x10h\xeb\x04\xe1\x0f\x92\x96\x1a\xc7\x8f\xe6\x8c\xe5\xa6+\x863b`"\xc7\xdc^6|v\x88\x7f\xdb\xb0\x1d\x7f0\x04\xe0S\x199{(\xae\x94\x1b\xd1\xac\xf2\xac\x1c\xb0\x94\x0cJp\xb0\xf1\x9e\\~8q 1V5\xda\x05\xed\xad\x7f`Q\x15\xc6\r\xea\xc7\xa2\x9cT\xfa\xc7\xd9()w#u#|\x93\x116\x14\xf0yLu\n\xf9\xcd\xd7R\x82?\xc8M\xd3G\xf3\xdaCy\xdc:e\x04\xfd\xe3b\xd0\x14\x85\xc4\x18;S/\x1bHK{\x80\xed\xfb\x8e\x81\xe0\x1b\x05E\xad4\rJ\x9c\x15\x08!\x04\x1f\xdc?\x9e)C\xe2\xbb\xb5rO_r\x9b+)\xc6\xc2\xb2\x99\x99\x1c^5\x9d\t\x83\xe3 \x14n\xf3*\nxN\x98\xc9\xa0\x1e\x0exRb\xb7j\xac{(\x8f?,\xbd\x9a\x9c\xf4\xa4\xe8Y]\xcf\xbcf\xc2\x10\x12R\xe2!\xd0Q\xa9\xa0h&\x90\xf0\x06g4\xc2\x90\xc1\xbf\xe7e\x91;\xd0\x15\xddT\x9c\x83!K)\x19\x9c`\xe7\x8b\x9f~\x9f\x9f\xe7g\x13g\x05\xea\x8f@Km\xd4\xee\xac\x9a\xc2\x15\xc3\x07\xf0\xf9\x13w\xb2d\xfe8,\x9a\xda\xe3\xac\xc8\xdf\xf4\x04\x97L\x8dsA\xc8\xbfK\xa3\xbd\xb9\x03G\xdc\x1e\xa41\xee\xcc\xadMHK\xb0\xb3\xfa\xaa\xacs\x06\xd0\xbd\xc3c\xde\xe6/\xf2\x861g\xb0\x9d\x17\x8a\xfcl\xaf\x80@\xc8`tz\x12\xb7^q\x19\x0f\xce\xbd\xbc\xdb\xcd\x9f\xee,\x87U\xc5*$\xa8\x96\x8f5\xbc\x8d>\xe2\x07}H\xd0?\x0e\x8b\xed\xf4v\x122\xb8}|\x1a\xaa"\xce\xa8N\xe7\x02f\xe6\xa8AL\xcaJ\xe5h\xa3\x07\xdd0\x18\x98\x10KB\xac\xfd[U6\xb2\xdauEH#\xd4\x82\xa1\xeeU\xd8\xfaj\x08\xbfg+\x81\xb63g[\x86d\xea\xd0\x84\xf3f\xa370\x00N\x9b\x85\x91\x83\x12\xc9\xc9H\x8e\x828\xcb3\xe4\t\x8f\x1f\x9f\xaf\xa3\x98\xc6\xea\x0e\xd3\xe76T\x96\x91\x90\xbe\x17)\xc7\x9c6\x80II\xb2\xd3\xd2\xe7.hDH\xdd\x90\x1c\xacu\xf3R\xc9Wl\xd8q\x08\xab\xaa\xf2\x93\x05\xe3\xb9gf\x0e\x89\xb1\xf6\xb3aD\x1c\xabk\xa0\xaa\xee\xe4\xfb\xecx\xc5g\x02\xd9\xfa\xda\x11R\x87\x97\xe0J\x19\x8d\xa2\x89^\x99\x11\x82\x86\xb6`\x9f@D\x04\xab\xa8;\xc9K%e\xfc\xae\xf8K\xda=\x1d\xa0\x99\xed\xb5\xe5/\x17\xb3i\xcf\x11\xd6\xdd;\x87\x91\x03\x13M\xe0B\xf4\x10FJI dp\xa4\xaei\x7fp\xfb\xe7\xa5\x00Jt\x90\xd9P\xf1\x9fx\x1a\xda\xa2>\xbcG\x1a \xf80\xdc\xdb\x95R\x9e\xb7:\xfdj\xe3\x0e\xf2\xd7\xbcK\xc1{;i\xef\x08\x98 "\x91\xddi\xa3d\xcf\x11\xae]\xfb.\xa5\x07\x8f\x9b\xc0Os\x96\xd7\x170\xde(\xdaQD\xf5\xdb\x87\xcdNc\xa4\xd7[\xb5\xbb\x81\x01\xa3\xd2I\xcc\x98\x8c\xa2\xc8h\x05\x16Y\x8a`\xd7\xb1VV\xce\x1e\x82US\xce\xca\x18\xbb~\xa6\xf4`\r\xf9\xbf\xd9\xc8\x86\x9d\x87h\xf2t\x80\x94\xb8\x9c6r\x87\xa4\x90\xd9?\x0e_ \x14\x05\xe6\xf6\xfa\xd8\xf2\xd5Q\xa6\x8d\x18Dz\x92\xab\xdb>\xe1\xe7\xf2\x95\xe2]-\xeb\xd7\xac_L\xfb\xc1F\xf2V\x9c\xc2\xda\xe5\xf3,\x0c\x9bYK\xc6\xe5Ih\xb6\x9e\x92\xea\x06\xf3F&\xf3\xee}\xe3p\xf62\xff8\x15\\ \xa4Ss\xb2\x8d_\xbe\xb3\x9dW\x8b\xf6\x80U\x03!\xc8\xee\x1f\xcf3w\xcfb\xe1\x84\xacn\xdf\x7ff\xd3.\xfe\xf5\x8f\xff\x8b?\xa4\x83n02#\x89w\x1e\xb9\x96\xd1\xe1\x80\x18I\x1b\xf7V70\xf6\xfe\'\xd7P\\\xf03\xf2V\xc2\x96\xb5]{\xbf+\xa0t\xbdA\xe6\xb8]\xf8=w!\x10\x18\xba\xc4\xd0\x05F\x08\xf4\x10\xc8\x10\x87k\xdc\x94\xec?N\xbaK%#\xc9\x85\xd6\xa5y\x1d\x01\xd1\xd0\xda\xce\xd6\x03\xc7y~\xcb\x1e\xee{\xa9\x88\xcf\x0f\x1c\x07\x9b\x05$\xdc~\xe5H6\xfd\xecFl\x16\x957\xff\xb6\x8f?\x94\x1e\xa0\xb8\xac\n\xb7\xd7\xc7=3G\xb3\xb7\xba\x89}\xc7\xdd\xa0\x08\x1a\x1b=\xfc\xad\xbc\x961\xe9Id\xa6\xf4\x03\xe0\xa3\x83\'\xc4\xddk\xff\xf8I\xd3\x9f\x7f~\x17\x00\xc3\xa7@\xc5\xf6S\x18\x89\xccF\x16\xaeZ\x8d\xc3\xf5+4\x9b\x99\xf7D\xab\xb3p7C\xd7M\xb5\x18\x9c\xcc\x98A\x89$\xba\x1ch\x9a\x8a\xdf\x1f\xa4\xb2\xbe\x85\xc3\r\xad\x94\xd7\x9d\xc4\xe3\xf1\x99\xe37! \xa43/w\x08E\xabo\xe6\xad\xd2\x03,_\xff)u\x8d\xad\x9d\x9ak\xd5x\xfb\xe1\x85\xe8Rr\xebs\x1ft\x89_:\xa9\x89\xb1\x8c\xcfL\x95zL\xb2\xd8{\xa0\xfcH\xdd\xde=3\xd8\xba\xeeX\xd71yw a\x9a\x98\xbb\xd4\x89#\xeeu\x9c\xf1\xb7D\x07\x95g0\xe2Nu\x92\x18\x86\xec=\xb35$)qN&g\xf5\xa7\xb0\xac\n]7:\xff\x1f6\xe8\xf4\xc4X\xfc\xbaNC\xab\xafg\x91\xd5/\r\xf4\x90\x87\xda\x03\xd7\xb2\xe97\x9f\x92\xb7\x02\xb6\x14\x9ca\x18\x1a\x19\xbbM\xbd\xd7E\xd2\xc0\xcd8\x13\xa6\x86=\x99\t\xe8l\x83\xa1\x94\x9d\x85\x90\xe8\x92\xe9\x1a\xf2\xf4\xb3\x94\x88\x87\x92\xd2L&#\x13_W\xaa@\xb5z8\xb2\xf3\xa7\x14\x16\xbc\xd2\xdbW{\xee\xb8y-\xe4\xad\x80\xd2\xd7<4\x1e\xcf\xa7\xad\xf9/\xe1\xacT\x84\xd99+\x10\x0e\xab\x85I\xc3\xd3\xb0j*\x04\xf5N\x17\xdb\x15DW\xc1u\x03\x02!\x92\\\x0e\xee\x9c>\n\x0cC\xa2j\x92\xb8\x01\x02\xc3h\xe1\xc8\x8eeQ\x10\xbd\xfc\xee\xab\xf7ah\xe56\xc8[\x0e\x9f\xbc\x10`p\xee\xfb\xe6\xcd\x8a\xe9\xa8\x96\xb3\xa3C\xc2\xfc\xdc!lxd!\xb3G\xa5S\xef\xf5QY{\x12:\xfc\xa6\xc0\xba\x0e!\xddL\xe5\x03A\xd0%\x99\x03\x13y`n.O\xdf5\x93E\x13\x87\xf1\xf4\xa62A|\x9a\xa0\xad\xe9\x10\xf5\xe5w\xb2\xb9`cd\x84@\xe1\xda^\x86\'gZ\x11c\x9a\xbfTA\xb1/\xc2\xee|\x1dG\\?\xa0g\x9c9\x85\x91\xac\xfe\xf1\xbc\xbfb\x119\xe9I\xf8\x02!Z;\x02\xec?\xee\xa6\xba\xa9\x15\xb7\xc7\x87aH\xfa\xc5X\xc9L\x89#;-\x9e\xfeqN\xec\x16\x8d\x17\x8bw\xcbU\x9b\xca\xc5I\xbf\x02\xf5\xe5\xef\xe3mZ\xcc\x96\xa7j\xba\xc9\xd3\xeb\x14\xe8\xdbV\xfe\n\x90\n\x14>\t\xb3\x7f\x9c\x8c3\xf65\x1cqW\xa1Y\x1d\xdd:~\xa7\x023\x0c\x84\x10\xac\xbcv\x12\x0f\xcc\x19K\x8a\xcb\x81USP\x15%j\xdf\xbaa\x10\xd4\r\xdcm~\xde\xff\xe20\xcf\x17\xef\xa5\xd2c\t\xe1on\xa2v\xdf\xe3\x14\xff\xf6w\xe1\xc1,l\xe9\xcb/\x1f\xba\xb1\xd3\x85\xd2\xf9\xcb\xe6a\x8fy\x00\x8bc\x06\x8a\x9a\x86\xaau\x02\xea\xda\x97\x95\x12\x02!\x90\x92!\xe9I\x0cK\x8d\'\xc6n&\x9e\x1d\x81\xa0lm\x0f\x88\x1ao\x90\xea\x9a\x16\xd3\xb8m\xf6]\xb4T\x7f\xc07\xbb\x9f\xe5\xcb\xbf\xb8\xcf\xa4J\xe7\x0f\x04\xe8\xe6\xf2.\x9b\xa90db.\xf6\x98\x19\x08\xe5\x1aTm\x0e\x8aEE\xb3t\x8f;]\xca\x80hW]\xb3\x85\xdf\x0b\x814\xaa\x11\xca\x16\x8c\xc0fj\x0el\xa7\xf4\xf5\xda\x1eg\x9d\xd5\x80\xf1|\xd6\xfceP\xf4\xb4\xf9|\xd6\x83\x02\xab\xd3\x81\xd5\x11\x83\x94\x93\x90\xc6,T\xcb8\x145\x1bEI\x02\xe1D\x08\x15)C\x08\xc5\x83P\xeb\x08u\xec#\x14\xdc\x89\xc5\xbe\x05\xa8\xa2\xb9\xa6\x83O^\n\xf4\xa5D\xf8\x7fM\xdd#\x1d\xf5i\xe0U\x00\x00\x00\x00IEND\xaeB`\x82' - fqfn = os.path.join(media_dir, 'image.png') + image_content = open(os.path.join(DIR, 'fixtures/test_image.png'), 'rb').read() + fqfn = os.path.join(self.media_dir, 'image.png') open(fqfn, 'wb').write(image_content) - hashes = set() + # Calculate hash from various sources and confirm they all match. + expected_hash = '35830221efe45ab0dc3d91ca23c29d2d3c20d00c9afeaa096ab256ec322a7a0b3293f07a01377e31060e65b4e5f6f8fdb4c0e56bc586bba5a7ab3e6d6d97a192' h = utils.get_text_hash(image_content) - hashes.add(h) - if verbose: print h + self.assertEqual(h, expected_hash) h = utils.get_file_hash(fqfn) - hashes.add(h) - if verbose: print h + self.assertEqual(h, expected_hash) h = utils.get_text_hash(open(fqfn, 'rb').read()) - hashes.add(h) - if verbose: print h - h = utils.get_text_hash(open(fqfn, 'r').read()) - hashes.add(h) - if verbose: print h - #print 'Hashes:', len(hashes) + self.assertEqual(h, expected_hash) +# h = utils.get_text_hash(open(fqfn, 'r').read())#not supported in py3 +# self.assertEqual(h, expected_hash) # Create test file. - self.assertEqual(len(hashes), 1) - image_content = u'aあä' - fqfn = os.path.join(media_dir, 'test.txt') + if six.PY3: + image_content = six.text_type('aあä')#, encoding='utf-8') + else: + image_content = six.text_type('aあä', encoding='utf-8') + fqfn = os.path.join(self.media_dir, 'test.txt') open(fqfn, 'wb').write(image_content.encode('utf-8')) - hashes = set() + expected_hash = '1f40fc92da241694750979ee6cf582f2d5d7d28e18335de05abc54d0560e0f5302860c652bf08d560252aa5e74210546f369fbbbce8c12cfc7957b2652fe9a75' h = utils.get_text_hash(image_content) - hashes.add(h) - if verbose: print h + self.assertEqual(h, expected_hash) h = utils.get_file_hash(fqfn) - hashes.add(h) - if verbose: print h + self.assertEqual(h, expected_hash) h = utils.get_text_hash(open(fqfn, 'rb').read()) - hashes.add(h) - if verbose: print h - h = utils.get_text_hash(open(fqfn, 'r').read()) - hashes.add(h) - if verbose: print h - #print 'Hashes:', len(hashes) - - self.assertEqual(len(hashes), 1) + self.assertEqual(h, expected_hash) +# h = utils.get_text_hash(open(fqfn, 'r').read()) +# self.assertEqual(h, expected_hash) class DatabaseFilesViewTestCase(TestCase): fixtures = ['test_data.json'] @@ -141,6 +135,6 @@ class DatabaseFilesViewTestCase(TestCase): def test_reading_file(self): self.assertEqual(File.objects.count(), 1) response = self.client.get('/files/1.txt') - self.assertEqual(response.content, '1234567890') + self.assertEqual(response.content, b'1234567890') self.assertEqual(response['content-type'], 'text/plain') - self.assertEqual(unicode(response['content-length']), '10') + self.assertEqual(response['content-length'], '10') diff --git a/database_files/urls.py b/database_files/urls.py index c256013..d438f75 100644 --- a/database_files/urls.py +++ b/database_files/urls.py @@ -1,4 +1,9 @@ -from django.conf.urls.defaults import * + +try: + # Removed in Django 1.6 + from django.conf.urls.defaults import patterns, url +except ImportError: + from django.conf.urls import patterns, url urlpatterns = patterns('', url(r'^files/(?P.+)$', 'database_files.views.serve', name='database_file'), diff --git a/database_files/utils.py b/database_files/utils.py index 488dedd..b43ea15 100644 --- a/database_files/utils.py +++ b/database_files/utils.py @@ -3,12 +3,9 @@ import os import hashlib -from django.conf import settings +import six -DEFAULT_ENFORCE_ENCODING = getattr(settings, 'DB_FILES_DEFAULT_ENFORCE_ENCODING', True) -DEFAULT_ENCODING = getattr(settings, 'DB_FILES_DEFAULT_ENCODING', 'ascii') -DEFAULT_ERROR_METHOD = getattr(settings, 'DB_FILES_DEFAULT_ERROR_METHOD', 'ignore') -DEFAULT_HASH_FN_TEMPLATE = getattr(settings, 'DB_FILES_DEFAULT_HASH_FN_TEMPLATE', '%s.hash') +from django.conf import settings def is_fresh(name, content_hash): """ @@ -48,7 +45,7 @@ def get_hash_fn(name): os.makedirs(dirs) fqfn_parts = os.path.split(fqfn) hash_fn = os.path.join(fqfn_parts[0], - DEFAULT_HASH_FN_TEMPLATE % fqfn_parts[1]) + settings.DB_FILES_DEFAULT_HASH_FN_TEMPLATE % fqfn_parts[1]) return hash_fn def write_file(name, content, overwrite=False): @@ -67,7 +64,12 @@ def write_file(name, content, overwrite=False): # Cache hash. hash = get_file_hash(fqfn) hash_fn = get_hash_fn(name) - open(hash_fn, 'wb').write(hash) + try: + # Write out bytes in Python3. + value = six.binary_type(hash, 'utf-8') + except TypeError: + value = hash + open(hash_fn, 'wb').write(value) # Set ownership and permissions. uname = getattr(settings, 'DATABASE_FILES_USER', None) @@ -82,36 +84,24 @@ def write_file(name, content, overwrite=False): if perms: os.system('chmod -R %s "%s"' % (perms, dirs)) -#def get_file_hash(fin): -# """ -# Iteratively builds a file hash without loading the entire file into memory. -# """ -# if isinstance(fin, basestring): -# fin = open(fin) -# h = hashlib.sha512() -# for text in fin.readlines(): -# if not isinstance(text, unicode): -# text = unicode(text, encoding='utf-8', errors='replace') -# h.update(text.encode('utf-8', 'replace')) -# return h.hexdigest() - def get_file_hash(fin, - force_encoding=DEFAULT_ENFORCE_ENCODING, - encoding=DEFAULT_ENCODING, - errors=DEFAULT_ERROR_METHOD): + force_encoding=settings.DB_FILES_DEFAULT_ENFORCE_ENCODING, + encoding=settings.DB_FILES_DEFAULT_ENCODING, + errors=settings.DB_FILES_DEFAULT_ERROR_METHOD, + chunk_size=128): """ Iteratively builds a file hash without loading the entire file into memory. """ - if isinstance(fin, basestring): - fin = open(fin, 'r') + if isinstance(fin, six.string_types): + fin = open(fin, 'rb') h = hashlib.sha512() while 1: - text = fin.read(1000) + text = fin.read(chunk_size) if not text: break if force_encoding: - if not isinstance(text, unicode): - text = unicode(text, encoding=encoding, errors=errors) + if not isinstance(text, six.text_type): + text = six.text_type(text, encoding=encoding, errors=errors) h.update(text.encode(encoding, errors)) else: h.update(text) @@ -122,22 +112,22 @@ def get_text_hash_0004(text): Returns the hash of the given text. """ h = hashlib.sha512() - if not isinstance(text, unicode): - text = unicode(text, encoding='utf-8', errors='replace') + if not isinstance(text, six.text_type): + text = six.text_type(text, encoding='utf-8', errors='replace') h.update(text.encode('utf-8', 'replace')) return h.hexdigest() def get_text_hash(text, - force_encoding=DEFAULT_ENFORCE_ENCODING, - encoding=DEFAULT_ENCODING, - errors=DEFAULT_ERROR_METHOD): + force_encoding=settings.DB_FILES_DEFAULT_ENFORCE_ENCODING, + encoding=settings.DB_FILES_DEFAULT_ENCODING, + errors=settings.DB_FILES_DEFAULT_ERROR_METHOD): """ Returns the hash of the given text. """ h = hashlib.sha512() if force_encoding: - if not isinstance(text, unicode): - text = unicode(text, encoding=encoding, errors=errors) + if not isinstance(text, six.text_type): + text = six.text_type(text, encoding=encoding, errors=errors) h.update(text.encode(encoding, errors)) else: h.update(text) diff --git a/database_files/views.py b/database_files/views.py index 7946b11..d0ea433 100644 --- a/database_files/views.py +++ b/database_files/views.py @@ -18,7 +18,7 @@ def serve(request, name): f = get_object_or_404(File, name=name) f.dump() mimetype = mimetypes.guess_type(name)[0] or 'application/octet-stream' - response = HttpResponse(f.content, mimetype=mimetype) + response = HttpResponse(f.content, content_type=mimetype) response['Content-Length'] = f.size return response diff --git a/setup.py b/setup.py index d7b5cfd..f19d1d7 100644 --- a/setup.py +++ b/setup.py @@ -1,43 +1,90 @@ #!/usr/bin/env python # -*- coding: utf-8 -*- import os -from distutils.core import setup, Command +from setuptools import setup, find_packages, Command + import database_files -def get_reqs(reqs=[]): - # optparse is included with Python <= 2.7, but has been deprecated in favor - # of argparse. We try to import argparse and if we can't, then we'll add - # it to the requirements - try: - import argparse - except ImportError: - reqs.append("argparse>=1.1") +def get_reqs(test=False): + reqs = [ + 'Django>=1.4', + 'six>=1.7.2', + ] + if test: + reqs.append('South>=0.8.4') return reqs class TestCommand(Command): description = "Runs unittests." - user_options = [] + user_options = [ + ('name=', None, + 'Name of the specific test to run.'), + ('virtual-env-dir=', None, + 'The location of the virtual environment to use.'), + ('pv=', None, + 'The version of Python to use. e.g. 2.7 or 3'), + ] + def initialize_options(self): - pass + self.name = None + self.virtual_env_dir = './.env%s' + self.pv = 0 + self.versions = [2.7, 3] + def finalize_options(self): pass + + def build_virtualenv(self, pv): + virtual_env_dir = self.virtual_env_dir % pv + kwargs = dict(virtual_env_dir=virtual_env_dir, pv=pv) + if not os.path.isdir(virtual_env_dir): + cmd = 'virtualenv -p /usr/bin/python{pv} {virtual_env_dir}'.format(**kwargs) + #print(cmd) + os.system(cmd) + + cmd = '. {virtual_env_dir}/bin/activate; easy_install -U distribute; deactivate'.format(**kwargs) + os.system(cmd) + + for package in get_reqs(test=True): + kwargs['package'] = package + cmd = '. {virtual_env_dir}/bin/activate; pip install -U {package}; deactivate'.format(**kwargs) + #print(cmd) + os.system(cmd) + def run(self): - os.system('django-admin.py test --pythonpath=. --settings=database_files.tests.settings tests') - #os.system('django-admin.py test --pythonpath=. --settings=database_files.tests.settings tests.DatabaseFilesTestCase.test_adding_file') + versions = self.versions + if self.pv: + versions = [self.pv] + + for pv in versions: + + self.build_virtualenv(pv) + kwargs = dict(pv=pv, name=self.name) + + if self.name: + cmd = '. ./.env{pv}/bin/activate; django-admin.py test --pythonpath=. --settings=database_files.tests.settings database_files.tests.tests.{name}; deactivate'.format(**kwargs) + else: + cmd = '. ./.env{pv}/bin/activate; django-admin.py test --pythonpath=. --settings=database_files.tests.settings database_files.tests; deactivate'.format(**kwargs) + + print(cmd) + ret = os.system(cmd) + if ret: + return setup( name='django-database-files', version=database_files.__version__, description='A storage system for Django that stores uploaded files in both the database and file system.', - author='Chris Spencer', - author_email='chrisspen@gmail.com', - url='http://github.com/chrisspen/django-database-files', + author='Ben Firshman', + author_email='ben@firshman.co.uk', + url='http://github.com/bfirsh/django-database-files/', packages=[ 'database_files', 'database_files.management', 'database_files.management.commands', 'database_files.migrations', ], + #https://pypi.python.org/pypi?%3Aaction=list_classifiers classifiers=[ 'Development Status :: 4 - Beta', 'Framework :: Django', @@ -45,9 +92,12 @@ def run(self): 'License :: OSI Approved :: BSD License', 'Operating System :: OS Independent', 'Programming Language :: Python', + 'Programming Language :: Python', + 'Programming Language :: Python :: 2.7', + 'Programming Language :: Python :: 3.0', ], - requires = ["Django (>=1.4)",], + install_requires = get_reqs(), cmdclass={ 'test': TestCommand, }, -) \ No newline at end of file +) From 624bf584620699c5c977624c08292f46cba738d3 Mon Sep 17 00:00:00 2001 From: chris Date: Wed, 18 Jun 2014 19:33:12 -0400 Subject: [PATCH 34/82] Updated version. --- database_files/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/database_files/__init__.py b/database_files/__init__.py index 868a985..5af465a 100644 --- a/database_files/__init__.py +++ b/database_files/__init__.py @@ -1,2 +1,2 @@ -VERSION = (0, 2, 0) +VERSION = (0, 2, 1) __version__ = '.'.join(map(str, VERSION)) \ No newline at end of file From 802b86cb1ad4ecd8031779289cb0ac579c16419d Mon Sep 17 00:00:00 2001 From: chris Date: Wed, 9 Jul 2014 14:21:18 -0400 Subject: [PATCH 35/82] Repackaged. Fixed Python3 unittest due to South bug. Updated docs. --- README.md | 63 +++++++++++++++++++++++------------ database_files/__init__.py | 2 +- database_files/tests/tests.py | 6 +++- database_files/urls.py | 7 +++- database_files/views.py | 8 +++-- setup.py | 60 ++++++++++++++++++++++++--------- 6 files changed, 102 insertions(+), 44 deletions(-) diff --git a/README.md b/README.md index 922d61e..4b49848 100644 --- a/README.md +++ b/README.md @@ -1,31 +1,16 @@ -django-database-files -===================== +Django Database Files 3000 +========================== -django-database-files is a storage system for Django that stores uploaded files -in the database. - -WARNING: It is generally a bad idea to serve static files from Django, -but there are some valid use cases. If your Django app is behind a caching -reverse proxy and you need to scale your application servers, it may be -simpler to store files in the database. +This is a storage system for Django that stores uploaded +files in the database. Files can be served from the database +(usually a bad idea), the file system, or a CDN. Installation ------------ - python setup.py install - -Or via pip with: +Simply install via pip with: - pip install django-database-files - -You can run unittests with: - - python setup.py test - -You can run unittests for a specific Python version using the `pv` parameter -like: - - python setup.py test --pv=3 + pip install django-database-files-3000 Usage ----- @@ -58,3 +43,37 @@ To delete all files in the database and file system not referenced by any model fields, run: python manage.py database_files_cleanup + +Settings +------- + +* `DB_FILES_AUTO_EXPORT_DB_TO_FS` = `True`|`False` (default `True`) + + If true, when a file is uploaded or read from the database, a copy will be + exported to your media directory corresponding to the FileField's upload_to + path, just as it would with the default Django file storage. + + If false, the file will only exist in the database. + +* `DATABASE_FILES_URL_METHOD` = `URL_METHOD_1`|`URL_METHOD_1` (default `URL_METHOD_1`) + + Defines the method to use when rendering the web-accessible URL for a file. + + If `URL_METHOD_1`, assumes all files have been exported to the filesystem and + uses the path corresponding to your `settings.MEDIA_URL`. + + If `URL_METHOD_2`, uses the URL bound to the `database_file` view + to dynamically lookup and serve files from the filesystem or database. + +Development +----------- + +You can run unittests with: + + python setup.py test + +You can run unittests for a specific Python version using the `pv` parameter +like: + + python setup.py test --pv=3.2 + \ No newline at end of file diff --git a/database_files/__init__.py b/database_files/__init__.py index 5af465a..d7e5a87 100644 --- a/database_files/__init__.py +++ b/database_files/__init__.py @@ -1,2 +1,2 @@ -VERSION = (0, 2, 1) +VERSION = (0, 3, 0) __version__ = '.'.join(map(str, VERSION)) \ No newline at end of file diff --git a/database_files/tests/tests.py b/database_files/tests/tests.py index c2dabaa..144917a 100644 --- a/database_files/tests/tests.py +++ b/database_files/tests/tests.py @@ -135,6 +135,10 @@ class DatabaseFilesViewTestCase(TestCase): def test_reading_file(self): self.assertEqual(File.objects.count(), 1) response = self.client.get('/files/1.txt') - self.assertEqual(response.content, b'1234567890') + if hasattr(response, 'streaming_content'): + content = list(response.streaming_content)[0] + else: + content = response.content + self.assertEqual(content, b'1234567890') self.assertEqual(response['content-type'], 'text/plain') self.assertEqual(response['content-length'], '10') diff --git a/database_files/urls.py b/database_files/urls.py index d438f75..4f508a9 100644 --- a/database_files/urls.py +++ b/database_files/urls.py @@ -6,5 +6,10 @@ from django.conf.urls import patterns, url urlpatterns = patterns('', - url(r'^files/(?P.+)$', 'database_files.views.serve', name='database_file'), +# url(r'^files/(?P.+)$', +# 'database_files.views.serve', +# name='database_file'), + url(r'^files/(?P.+)$', + 'database_files.views.serve_mixed', + name='database_file'), ) diff --git a/database_files/views.py b/database_files/views.py index d0ea433..a7f971f 100644 --- a/database_files/views.py +++ b/database_files/views.py @@ -1,6 +1,7 @@ import base64 import os +from django.conf import settings from django.http import Http404, HttpResponse from django.shortcuts import get_object_or_404 from django.views.decorators.cache import cache_control @@ -22,15 +23,16 @@ def serve(request, name): response['Content-Length'] = f.size return response -def serve_mixed(request, path, document_root): +def serve_mixed(request, name, document_root=None): """ First attempts to serve the file from the filesystem, then tries the database. """ + document_root = document_root or settings.MEDIA_ROOT try: # First attempt to serve from filesystem. - return django_serve(request, path, document_root) + return django_serve(request, name, document_root) except Http404: # Then try serving from database. - return serve(request, path) + return serve(request, name) \ No newline at end of file diff --git a/setup.py b/setup.py index f19d1d7..cd716d5 100644 --- a/setup.py +++ b/setup.py @@ -5,13 +5,26 @@ import database_files -def get_reqs(test=False): +try: + from pypandoc import convert + read_md = lambda f: convert(f, 'rst') +except ImportError: + print("Warning: pypandoc module not found, could not convert " + "Markdown to RST") + read_md = lambda f: open(f, 'r').read() + +def get_reqs(test=False, pv=None): reqs = [ 'Django>=1.4', 'six>=1.7.2', ] if test: - reqs.append('South>=0.8.4') + #TODO:remove once Django 1.7 south integration becomes main-stream? + if not pv: + reqs.append('South>=1.0') + elif pv <= 3.2: + # Note, South dropped Python3 support after 0.8.4... + reqs.append('South==0.8.4') return reqs class TestCommand(Command): @@ -27,9 +40,13 @@ class TestCommand(Command): def initialize_options(self): self.name = None - self.virtual_env_dir = './.env%s' + self.virtual_env_dir = '.env%s' self.pv = 0 - self.versions = [2.7, 3] + self.versions = [ + 2.7, + 3, + #3.3,#TODO? + ] def finalize_options(self): pass @@ -39,16 +56,17 @@ def build_virtualenv(self, pv): kwargs = dict(virtual_env_dir=virtual_env_dir, pv=pv) if not os.path.isdir(virtual_env_dir): cmd = 'virtualenv -p /usr/bin/python{pv} {virtual_env_dir}'.format(**kwargs) - #print(cmd) + print(cmd) os.system(cmd) - cmd = '. {virtual_env_dir}/bin/activate; easy_install -U distribute; deactivate'.format(**kwargs) + cmd = '{virtual_env_dir}/bin/easy_install -U distribute'.format(**kwargs) + print(cmd) os.system(cmd) - for package in get_reqs(test=True): + for package in get_reqs(test=True, pv=float(pv)): kwargs['package'] = package - cmd = '. {virtual_env_dir}/bin/activate; pip install -U {package}; deactivate'.format(**kwargs) - #print(cmd) + cmd = '{virtual_env_dir}/bin/pip install -U {package}'.format(**kwargs) + print(cmd) os.system(cmd) def run(self): @@ -59,25 +77,34 @@ def run(self): for pv in versions: self.build_virtualenv(pv) - kwargs = dict(pv=pv, name=self.name) + kwargs = dict( + pv=pv, + virtual_env_dir=self.virtual_env_dir % pv, + name=self.name) if self.name: - cmd = '. ./.env{pv}/bin/activate; django-admin.py test --pythonpath=. --settings=database_files.tests.settings database_files.tests.tests.{name}; deactivate'.format(**kwargs) + cmd = '{virtual_env_dir}/bin/django-admin.py test --pythonpath=. --settings=database_files.tests.settings database_files.tests.tests.{name}'.format(**kwargs) else: - cmd = '. ./.env{pv}/bin/activate; django-admin.py test --pythonpath=. --settings=database_files.tests.settings database_files.tests; deactivate'.format(**kwargs) + cmd = '{virtual_env_dir}/bin/django-admin.py test --pythonpath=. --settings=database_files.tests.settings database_files.tests'.format(**kwargs) print(cmd) ret = os.system(cmd) if ret: return +try: + long_description = read_md('README.md') +except IOError: + long_description = '' + setup( - name='django-database-files', + name='django-database-files-3000', version=database_files.__version__, description='A storage system for Django that stores uploaded files in both the database and file system.', - author='Ben Firshman', - author_email='ben@firshman.co.uk', - url='http://github.com/bfirsh/django-database-files/', + long_description=long_description, + author='Chris Spencer', + author_email='chrisspen@gmail.com', + url='http://github.com/chrisspen/django-database-files-3000', packages=[ 'database_files', 'database_files.management', @@ -95,6 +122,7 @@ def run(self): 'Programming Language :: Python', 'Programming Language :: Python :: 2.7', 'Programming Language :: Python :: 3.0', + 'Programming Language :: Python :: 3.2', ], install_requires = get_reqs(), cmdclass={ From 03e2d2b842b6a18b9c19fe063da2656283439e21 Mon Sep 17 00:00:00 2001 From: chris Date: Sat, 26 Jul 2014 23:36:44 -0400 Subject: [PATCH 36/82] Broadened exception handling for pypandoc. --- database_files/__init__.py | 2 +- setup.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/database_files/__init__.py b/database_files/__init__.py index d7e5a87..81657fc 100644 --- a/database_files/__init__.py +++ b/database_files/__init__.py @@ -1,2 +1,2 @@ -VERSION = (0, 3, 0) +VERSION = (0, 3, 1) __version__ = '.'.join(map(str, VERSION)) \ No newline at end of file diff --git a/setup.py b/setup.py index cd716d5..6d9dfdc 100644 --- a/setup.py +++ b/setup.py @@ -8,7 +8,7 @@ try: from pypandoc import convert read_md = lambda f: convert(f, 'rst') -except ImportError: +except: print("Warning: pypandoc module not found, could not convert " "Markdown to RST") read_md = lambda f: open(f, 'r').read() From a9b285ce91a72e373632dd2a857adb36b0971143 Mon Sep 17 00:00:00 2001 From: chris Date: Mon, 8 Dec 2014 19:49:42 -0500 Subject: [PATCH 37/82] Added error handling for cases where path is passed. --- database_files/__init__.py | 2 +- database_files/views.py | 4 +++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/database_files/__init__.py b/database_files/__init__.py index 81657fc..5d14052 100644 --- a/database_files/__init__.py +++ b/database_files/__init__.py @@ -1,2 +1,2 @@ -VERSION = (0, 3, 1) +VERSION = (0, 3, 2) __version__ = '.'.join(map(str, VERSION)) \ No newline at end of file diff --git a/database_files/views.py b/database_files/views.py index a7f971f..0c9dd83 100644 --- a/database_files/views.py +++ b/database_files/views.py @@ -23,11 +23,13 @@ def serve(request, name): response['Content-Length'] = f.size return response -def serve_mixed(request, name, document_root=None): +def serve_mixed(request, *args, **kwargs): """ First attempts to serve the file from the filesystem, then tries the database. """ + name = kwargs.get('name') or kwargs.get('path') + document_root = kwargs.get('document_root') document_root = document_root or settings.MEDIA_ROOT try: # First attempt to serve from filesystem. From 266bfece9e959cb49ca0651130ea62b6a13afa25 Mon Sep 17 00:00:00 2001 From: chris Date: Mon, 8 Dec 2014 20:49:10 -0500 Subject: [PATCH 38/82] Fixed error handling bug in setup. --- database_files/__init__.py | 2 +- setup.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/database_files/__init__.py b/database_files/__init__.py index 5d14052..f337265 100644 --- a/database_files/__init__.py +++ b/database_files/__init__.py @@ -1,2 +1,2 @@ -VERSION = (0, 3, 2) +VERSION = (0, 3, 3) __version__ = '.'.join(map(str, VERSION)) \ No newline at end of file diff --git a/setup.py b/setup.py index 6d9dfdc..6fac0ff 100644 --- a/setup.py +++ b/setup.py @@ -94,7 +94,7 @@ def run(self): try: long_description = read_md('README.md') -except IOError: +except: long_description = '' setup( From 8b6bda0fb6a5faa1cc3ca2ba66494a58c2af6260 Mon Sep 17 00:00:00 2001 From: chris Date: Tue, 24 Feb 2015 16:57:50 -0500 Subject: [PATCH 39/82] Added option to specify filename to cleanup. --- database_files/__init__.py | 2 +- .../management/commands/database_files_cleanup.py | 14 +++++++++++++- database_files/storage.py | 2 +- 3 files changed, 15 insertions(+), 3 deletions(-) diff --git a/database_files/__init__.py b/database_files/__init__.py index f337265..1667044 100644 --- a/database_files/__init__.py +++ b/database_files/__init__.py @@ -1,2 +1,2 @@ -VERSION = (0, 3, 3) +VERSION = (0, 3, 4) __version__ = '.'.join(map(str, VERSION)) \ No newline at end of file diff --git a/database_files/management/commands/database_files_cleanup.py b/database_files/management/commands/database_files_cleanup.py index af5f50e..5b9d532 100644 --- a/database_files/management/commands/database_files_cleanup.py +++ b/database_files/management/commands/database_files_cleanup.py @@ -21,6 +21,9 @@ class Command(BaseCommand): default=False, help='If given, only displays the names of orphaned files ' + \ 'and does not delete them.'), + make_option('--filenames', + default='', + help='If given, only files with these names will be checked'), ) def handle(self, *args, **options): @@ -28,6 +31,11 @@ def handle(self, *args, **options): settings.DEBUG = False names = set() dryrun = options['dryrun'] + filenames = set([ + _.strip() + for _ in options['filenames'].split(',') + if _.strip() + ]) try: for model in get_models(): print('Checking model %s...' % (model,)) @@ -50,9 +58,13 @@ def handle(self, *args, **options): if not file.name: continue names.add(file.name) + # Find all database files with names not in our list. print('Finding orphaned files...') - orphan_files = File.objects.exclude(name__in=names).only('name', 'size') + orphan_files = File.objects.exclude(name__in=names) + if filenames: + orphan_files = orphan_files.filter(name__in=filenames) + orphan_files = orphan_files.only('name', 'size') total_bytes = 0 orphan_total = orphan_files.count() orphan_i = 0 diff --git a/database_files/storage.py b/database_files/storage.py index 48a97b1..e4ea8bc 100644 --- a/database_files/storage.py +++ b/database_files/storage.py @@ -90,7 +90,7 @@ def exists(self, name): Returns true if a file with the given filename exists in the database. Returns false otherwise. """ - if models.File.objects.filter(name=name).count() > 0: + if models.File.objects.filter(name=name).exists(): return True return super(DatabaseStorage, self).exists(name) From 4fdbcb4e85705e8ca7b623b69f1e76ea944a2a98 Mon Sep 17 00:00:00 2001 From: Richard Xia Date: Sat, 1 Aug 2015 22:34:11 -0700 Subject: [PATCH 40/82] Move South migrations to south_migrations. --- database_files/{migrations => south_migrations}/0001_initial.py | 0 .../{migrations => south_migrations}/0002_load_files.py | 0 ...d_field_file_created_datetime__add_field_file__content_hash.py | 0 database_files/{migrations => south_migrations}/0004_set_hash.py | 0 database_files/{migrations => south_migrations}/0005_auto.py | 0 database_files/{migrations => south_migrations}/__init__.py | 0 6 files changed, 0 insertions(+), 0 deletions(-) rename database_files/{migrations => south_migrations}/0001_initial.py (100%) rename database_files/{migrations => south_migrations}/0002_load_files.py (100%) rename database_files/{migrations => south_migrations}/0003_auto__add_field_file_created_datetime__add_field_file__content_hash.py (100%) rename database_files/{migrations => south_migrations}/0004_set_hash.py (100%) rename database_files/{migrations => south_migrations}/0005_auto.py (100%) rename database_files/{migrations => south_migrations}/__init__.py (100%) diff --git a/database_files/migrations/0001_initial.py b/database_files/south_migrations/0001_initial.py similarity index 100% rename from database_files/migrations/0001_initial.py rename to database_files/south_migrations/0001_initial.py diff --git a/database_files/migrations/0002_load_files.py b/database_files/south_migrations/0002_load_files.py similarity index 100% rename from database_files/migrations/0002_load_files.py rename to database_files/south_migrations/0002_load_files.py diff --git a/database_files/migrations/0003_auto__add_field_file_created_datetime__add_field_file__content_hash.py b/database_files/south_migrations/0003_auto__add_field_file_created_datetime__add_field_file__content_hash.py similarity index 100% rename from database_files/migrations/0003_auto__add_field_file_created_datetime__add_field_file__content_hash.py rename to database_files/south_migrations/0003_auto__add_field_file_created_datetime__add_field_file__content_hash.py diff --git a/database_files/migrations/0004_set_hash.py b/database_files/south_migrations/0004_set_hash.py similarity index 100% rename from database_files/migrations/0004_set_hash.py rename to database_files/south_migrations/0004_set_hash.py diff --git a/database_files/migrations/0005_auto.py b/database_files/south_migrations/0005_auto.py similarity index 100% rename from database_files/migrations/0005_auto.py rename to database_files/south_migrations/0005_auto.py diff --git a/database_files/migrations/__init__.py b/database_files/south_migrations/__init__.py similarity index 100% rename from database_files/migrations/__init__.py rename to database_files/south_migrations/__init__.py From 452c8271c49d72003482eae063e9a69bf99f927e Mon Sep 17 00:00:00 2001 From: Richard Xia Date: Sat, 1 Aug 2015 22:53:08 -0700 Subject: [PATCH 41/82] Add Django-1.7-style migrations. --- database_files/migrations/0001_initial.py | 28 +++++++++++++++++++++++ database_files/migrations/__init__.py | 0 2 files changed, 28 insertions(+) create mode 100644 database_files/migrations/0001_initial.py create mode 100644 database_files/migrations/__init__.py diff --git a/database_files/migrations/0001_initial.py b/database_files/migrations/0001_initial.py new file mode 100644 index 0000000..3400004 --- /dev/null +++ b/database_files/migrations/0001_initial.py @@ -0,0 +1,28 @@ +# -*- coding: utf-8 -*- +from __future__ import unicode_literals + +from django.db import models, migrations +import django.utils.timezone + + +class Migration(migrations.Migration): + + dependencies = [ + ] + + operations = [ + migrations.CreateModel( + name='File', + fields=[ + ('id', models.AutoField(verbose_name='ID', serialize=False, auto_created=True, primary_key=True)), + ('name', models.CharField(unique=True, max_length=255, db_index=True)), + ('size', models.PositiveIntegerField(db_index=True)), + ('_content', models.TextField(db_column=b'content')), + ('created_datetime', models.DateTimeField(default=django.utils.timezone.now, verbose_name=b'Created datetime', db_index=True)), + ('_content_hash', models.CharField(db_index=True, max_length=128, null=True, db_column=b'content_hash', blank=True)), + ], + options={ + 'db_table': 'database_files_file', + }, + ), + ] diff --git a/database_files/migrations/__init__.py b/database_files/migrations/__init__.py new file mode 100644 index 0000000..e69de29 From 7e8e5bc1df8709465060f0854d32a6d951b598b4 Mon Sep 17 00:00:00 2001 From: Richard Xia Date: Mon, 24 Aug 2015 15:01:43 -0700 Subject: [PATCH 42/82] Add south_migrations to distribution. --- setup.py | 1 + 1 file changed, 1 insertion(+) diff --git a/setup.py b/setup.py index 6fac0ff..409528c 100644 --- a/setup.py +++ b/setup.py @@ -110,6 +110,7 @@ def run(self): 'database_files.management', 'database_files.management.commands', 'database_files.migrations', + 'database_files.south_migrations', ], #https://pypi.python.org/pypi?%3Aaction=list_classifiers classifiers=[ From 71b42d288b68b2bac07f8e423429971e68a7ffd5 Mon Sep 17 00:00:00 2001 From: Roger Hunwicks Date: Thu, 4 Feb 2016 10:55:44 +0600 Subject: [PATCH 43/82] Make Django 1.7 migrations compatible with Python3 --- database_files/migrations/0001_initial.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/database_files/migrations/0001_initial.py b/database_files/migrations/0001_initial.py index 3400004..62a2d4d 100644 --- a/database_files/migrations/0001_initial.py +++ b/database_files/migrations/0001_initial.py @@ -17,9 +17,9 @@ class Migration(migrations.Migration): ('id', models.AutoField(verbose_name='ID', serialize=False, auto_created=True, primary_key=True)), ('name', models.CharField(unique=True, max_length=255, db_index=True)), ('size', models.PositiveIntegerField(db_index=True)), - ('_content', models.TextField(db_column=b'content')), - ('created_datetime', models.DateTimeField(default=django.utils.timezone.now, verbose_name=b'Created datetime', db_index=True)), - ('_content_hash', models.CharField(db_index=True, max_length=128, null=True, db_column=b'content_hash', blank=True)), + ('_content', models.TextField(db_column='content')), + ('created_datetime', models.DateTimeField(default=django.utils.timezone.now, verbose_name='Created datetime', db_index=True)), + ('_content_hash', models.CharField(db_index=True, max_length=128, null=True, db_column='content_hash', blank=True)), ], options={ 'db_table': 'database_files_file', From 96d814448d7b73d2a0f0a5272c286b95b71154d0 Mon Sep 17 00:00:00 2001 From: Roger Hunwicks Date: Thu, 4 Feb 2016 11:23:26 +0600 Subject: [PATCH 44/82] Update README.md to include instructions for changing urls.py --- README.md | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/README.md b/README.md index 4b49848..c2890ba 100644 --- a/README.md +++ b/README.md @@ -65,6 +65,16 @@ Settings If `URL_METHOD_2`, uses the URL bound to the `database_file` view to dynamically lookup and serve files from the filesystem or database. + In this case, you will also need to updates your `urls.py` to include the view + that serves the files: + + urlpatterns = patterns('', + # ... the rest of your URLconf goes here ... + + # Serve Database Files directly + url(r'', include('database_files.urls')), + ) + Development ----------- From 32d45e20146b4b3af2f1a397f34efbda46973608 Mon Sep 17 00:00:00 2001 From: Chris Spencer Date: Tue, 11 Oct 2016 18:43:57 -0400 Subject: [PATCH 45/82] Added support for Django<=1.10.1. --- .travis.yml | 7 + MANIFEST.in | 5 + README.md | 29 ++- database_files/__init__.py | 2 +- database_files/migrations/0001_initial.py | 57 +++--- database_files/migrations/__init__.py | 21 ++ .../south_migrations/0001_initial.py | 38 ++++ .../0002_load_files.py | 0 ..._datetime__add_field_file__content_hash.py | 0 .../0004_set_hash.py | 0 .../0005_auto.py | 0 database_files/south_migrations/__init__.py | 0 .../{test_data.json => test_files.json} | 0 database_files/tests/settings.py | 14 +- database_files/tests/tests.py | 5 +- pip-requirements-min-django.txt | 2 + pip-requirements-test.txt | 1 + pip-requirements.txt | 2 + setup.py | 184 +++++++++--------- tox.ini | 32 +++ 20 files changed, 268 insertions(+), 131 deletions(-) create mode 100644 .travis.yml create mode 100644 MANIFEST.in create mode 100644 database_files/south_migrations/0001_initial.py rename database_files/{migrations => south_migrations}/0002_load_files.py (100%) rename database_files/{migrations => south_migrations}/0003_auto__add_field_file_created_datetime__add_field_file__content_hash.py (100%) rename database_files/{migrations => south_migrations}/0004_set_hash.py (100%) rename database_files/{migrations => south_migrations}/0005_auto.py (100%) create mode 100644 database_files/south_migrations/__init__.py rename database_files/tests/fixtures/{test_data.json => test_files.json} (100%) create mode 100644 pip-requirements-min-django.txt create mode 100644 pip-requirements-test.txt create mode 100644 pip-requirements.txt create mode 100644 tox.ini diff --git a/.travis.yml b/.travis.yml new file mode 100644 index 0000000..216627b --- /dev/null +++ b/.travis.yml @@ -0,0 +1,7 @@ +language: python +python: +- "3.5" +install: +- pip install tox +script: +- tox diff --git a/MANIFEST.in b/MANIFEST.in new file mode 100644 index 0000000..c60fca3 --- /dev/null +++ b/MANIFEST.in @@ -0,0 +1,5 @@ +recursive-include database_files/tests/fixtures * +recursive-include database_files/tests/media * +include pip-requirements-min-django.txt +include pip-requirements.txt +include pip-requirements-test.txt diff --git a/README.md b/README.md index 4b49848..91af5a3 100644 --- a/README.md +++ b/README.md @@ -68,12 +68,29 @@ Settings Development ----------- -You can run unittests with: +Tests require the Python development headers to be installed, which you can install on Ubuntu with: - python setup.py test + sudo apt-get install python-dev python3-dev python3.4-dev -You can run unittests for a specific Python version using the `pv` parameter -like: +To run unittests across multiple Python versions, install: - python setup.py test --pv=3.2 - \ No newline at end of file + sudo apt-get install python3.4-minimal python3.4-dev python3.5-minimal python3.5-dev + +Note, you may need to enable an [additional repository](https://launchpad.net/~fkrull/+archive/ubuntu/deadsnakes) to provide these packages. + +To run all [tests](http://tox.readthedocs.org/en/latest/): + + export TESTNAME=; tox + +To run tests for a specific environment (e.g. Python 2.7 with Django 1.4): + + export TESTNAME=; tox -e py27-django15 + +To run a specific test: + + export TESTNAME=.test_adding_file; tox -e py27-django15 + +To build and deploy a versioned package to PyPI, verify [all unittests are passing](https://travis-ci.org/chrisspen/django-database-files), and then run: + + python setup.py sdist + python setup.py sdist upload diff --git a/database_files/__init__.py b/database_files/__init__.py index 1667044..c6d9b06 100644 --- a/database_files/__init__.py +++ b/database_files/__init__.py @@ -1,2 +1,2 @@ -VERSION = (0, 3, 4) +VERSION = (1, 0, 0) __version__ = '.'.join(map(str, VERSION)) \ No newline at end of file diff --git a/database_files/migrations/0001_initial.py b/database_files/migrations/0001_initial.py index 010f933..1eda234 100644 --- a/database_files/migrations/0001_initial.py +++ b/database_files/migrations/0001_initial.py @@ -1,38 +1,31 @@ -# encoding: utf-8 -import datetime +# -*- coding: utf-8 -*- +# Generated by Django 1.9.1 on 2016-10-11 18:11 +from __future__ import unicode_literals -from django.core.management import call_command -from django.db import models +from django.db import migrations, models +import django.utils.timezone -from south.db import db -from south.v2 import SchemaMigration -class Migration(SchemaMigration): +class Migration(migrations.Migration): - def forwards(self, orm): - - # Adding model 'File' - db.create_table('database_files_file', ( - ('id', self.gf('django.db.models.fields.AutoField')(primary_key=True)), - ('name', self.gf('django.db.models.fields.CharField')(unique=True, max_length=255, db_index=True)), - ('size', self.gf('django.db.models.fields.PositiveIntegerField')()), - ('_content', self.gf('django.db.models.fields.TextField')(db_column='content')), - )) - db.send_create_signal('database_files', ['File']) + initial = True - def backwards(self, orm): - - # Deleting model 'File' - db.delete_table('database_files_file') + dependencies = [ + ] - models = { - 'database_files.file': { - 'Meta': {'object_name': 'File'}, - '_content': ('django.db.models.fields.TextField', [], {'db_column': "'content'"}), - 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), - 'name': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '255', 'db_index': 'True'}), - 'size': ('django.db.models.fields.PositiveIntegerField', [], {}) - } - } - - complete_apps = ['database_files'] + operations = [ + migrations.CreateModel( + name='File', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('name', models.CharField(db_index=True, max_length=255, unique=True)), + ('size', models.PositiveIntegerField(db_index=True)), + ('_content', models.TextField(db_column=b'content')), + ('created_datetime', models.DateTimeField(db_index=True, default=django.utils.timezone.now, verbose_name=b'Created datetime')), + ('_content_hash', models.CharField(blank=True, db_column=b'content_hash', db_index=True, max_length=128, null=True)), + ], + options={ + 'db_table': 'database_files_file', + }, + ), + ] diff --git a/database_files/migrations/__init__.py b/database_files/migrations/__init__.py index e69de29..de7bd50 100644 --- a/database_files/migrations/__init__.py +++ b/database_files/migrations/__init__.py @@ -0,0 +1,21 @@ +""" +Django migrations for chroniker app + +This package does not contain South migrations. South migrations can be found +in the ``south_migrations`` package. +""" + +SOUTH_ERROR_MESSAGE = """\n +For South support, customize the SOUTH_MIGRATION_MODULES setting like so: + + SOUTH_MIGRATION_MODULES = { + 'database_files': 'database_files.south_migrations', + } +""" + +# Ensure the user is not using Django 1.6 or below with South +try: + from django.db import migrations # noqa +except ImportError: + from django.core.exceptions import ImproperlyConfigured + raise ImproperlyConfigured(SOUTH_ERROR_MESSAGE) diff --git a/database_files/south_migrations/0001_initial.py b/database_files/south_migrations/0001_initial.py new file mode 100644 index 0000000..010f933 --- /dev/null +++ b/database_files/south_migrations/0001_initial.py @@ -0,0 +1,38 @@ +# encoding: utf-8 +import datetime + +from django.core.management import call_command +from django.db import models + +from south.db import db +from south.v2 import SchemaMigration + +class Migration(SchemaMigration): + + def forwards(self, orm): + + # Adding model 'File' + db.create_table('database_files_file', ( + ('id', self.gf('django.db.models.fields.AutoField')(primary_key=True)), + ('name', self.gf('django.db.models.fields.CharField')(unique=True, max_length=255, db_index=True)), + ('size', self.gf('django.db.models.fields.PositiveIntegerField')()), + ('_content', self.gf('django.db.models.fields.TextField')(db_column='content')), + )) + db.send_create_signal('database_files', ['File']) + + def backwards(self, orm): + + # Deleting model 'File' + db.delete_table('database_files_file') + + models = { + 'database_files.file': { + 'Meta': {'object_name': 'File'}, + '_content': ('django.db.models.fields.TextField', [], {'db_column': "'content'"}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'name': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '255', 'db_index': 'True'}), + 'size': ('django.db.models.fields.PositiveIntegerField', [], {}) + } + } + + complete_apps = ['database_files'] diff --git a/database_files/migrations/0002_load_files.py b/database_files/south_migrations/0002_load_files.py similarity index 100% rename from database_files/migrations/0002_load_files.py rename to database_files/south_migrations/0002_load_files.py diff --git a/database_files/migrations/0003_auto__add_field_file_created_datetime__add_field_file__content_hash.py b/database_files/south_migrations/0003_auto__add_field_file_created_datetime__add_field_file__content_hash.py similarity index 100% rename from database_files/migrations/0003_auto__add_field_file_created_datetime__add_field_file__content_hash.py rename to database_files/south_migrations/0003_auto__add_field_file_created_datetime__add_field_file__content_hash.py diff --git a/database_files/migrations/0004_set_hash.py b/database_files/south_migrations/0004_set_hash.py similarity index 100% rename from database_files/migrations/0004_set_hash.py rename to database_files/south_migrations/0004_set_hash.py diff --git a/database_files/migrations/0005_auto.py b/database_files/south_migrations/0005_auto.py similarity index 100% rename from database_files/migrations/0005_auto.py rename to database_files/south_migrations/0005_auto.py diff --git a/database_files/south_migrations/__init__.py b/database_files/south_migrations/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/database_files/tests/fixtures/test_data.json b/database_files/tests/fixtures/test_files.json similarity index 100% rename from database_files/tests/fixtures/test_data.json rename to database_files/tests/fixtures/test_files.json diff --git a/database_files/tests/settings.py b/database_files/tests/settings.py index 53d5758..185510e 100644 --- a/database_files/tests/settings.py +++ b/database_files/tests/settings.py @@ -27,8 +27,18 @@ MEDIA_ROOT = os.path.join(PROJECT_DIR, 'media') -# Run our South migrations during unittesting. -SOUTH_TESTS_MIGRATE = True +# Disable migrations. +# http://stackoverflow.com/a/28560805/247542 +class DisableMigrations(object): + + def __contains__(self, item): + return True + + def __getitem__(self, item): + return "notmigrations" +SOUTH_TESTS_MIGRATE = False # <= Django 1.8 +# if django.VERSION > (1, 7, 0): # > Django 1.8 +# MIGRATION_MODULES = DisableMigrations() USE_TZ = True diff --git a/database_files/tests/tests.py b/database_files/tests/tests.py index 144917a..4631744 100644 --- a/database_files/tests/tests.py +++ b/database_files/tests/tests.py @@ -8,6 +8,7 @@ from django.core import files from django.test import TestCase from django.core.files.storage import default_storage +from django.core.management import call_command from database_files.models import File from database_files.tests.models import Thing @@ -129,10 +130,8 @@ def test_hash(self): # h = utils.get_text_hash(open(fqfn, 'r').read()) # self.assertEqual(h, expected_hash) -class DatabaseFilesViewTestCase(TestCase): - fixtures = ['test_data.json'] - def test_reading_file(self): + call_command('loaddata', 'test_files.json') self.assertEqual(File.objects.count(), 1) response = self.client.get('/files/1.txt') if hasattr(response, 'streaming_content'): diff --git a/pip-requirements-min-django.txt b/pip-requirements-min-django.txt new file mode 100644 index 0000000..2362654 --- /dev/null +++ b/pip-requirements-min-django.txt @@ -0,0 +1,2 @@ +Django>=1.4 +Django<2 diff --git a/pip-requirements-test.txt b/pip-requirements-test.txt new file mode 100644 index 0000000..cb12429 --- /dev/null +++ b/pip-requirements-test.txt @@ -0,0 +1 @@ +tox>=2.0.0 diff --git a/pip-requirements.txt b/pip-requirements.txt new file mode 100644 index 0000000..b9700c2 --- /dev/null +++ b/pip-requirements.txt @@ -0,0 +1,2 @@ +six>=1.7.2 +South \ No newline at end of file diff --git a/setup.py b/setup.py index 6fac0ff..36cf2f5 100644 --- a/setup.py +++ b/setup.py @@ -5,92 +5,104 @@ import database_files -try: - from pypandoc import convert - read_md = lambda f: convert(f, 'rst') -except: - print("Warning: pypandoc module not found, could not convert " - "Markdown to RST") - read_md = lambda f: open(f, 'r').read() +# try: +# from pypandoc import convert +# read_md = lambda f: convert(f, 'rst') +# except: +# print("Warning: pypandoc module not found, could not convert " +# "Markdown to RST") +# read_md = lambda f: open(f, 'r').read() + +CURRENT_DIR = os.path.abspath(os.path.dirname(__file__)) -def get_reqs(test=False, pv=None): - reqs = [ - 'Django>=1.4', - 'six>=1.7.2', - ] - if test: - #TODO:remove once Django 1.7 south integration becomes main-stream? - if not pv: - reqs.append('South>=1.0') - elif pv <= 3.2: - # Note, South dropped Python3 support after 0.8.4... - reqs.append('South==0.8.4') - return reqs +def get_reqs(*fns): + lst = [] + for fn in fns: + for package in open(os.path.join(CURRENT_DIR, fn)).readlines(): + package = package.strip() + if not package: + continue + lst.append(package.strip()) + return lst -class TestCommand(Command): - description = "Runs unittests." - user_options = [ - ('name=', None, - 'Name of the specific test to run.'), - ('virtual-env-dir=', None, - 'The location of the virtual environment to use.'), - ('pv=', None, - 'The version of Python to use. e.g. 2.7 or 3'), - ] - - def initialize_options(self): - self.name = None - self.virtual_env_dir = '.env%s' - self.pv = 0 - self.versions = [ - 2.7, - 3, - #3.3,#TODO? - ] - - def finalize_options(self): - pass - - def build_virtualenv(self, pv): - virtual_env_dir = self.virtual_env_dir % pv - kwargs = dict(virtual_env_dir=virtual_env_dir, pv=pv) - if not os.path.isdir(virtual_env_dir): - cmd = 'virtualenv -p /usr/bin/python{pv} {virtual_env_dir}'.format(**kwargs) - print(cmd) - os.system(cmd) - - cmd = '{virtual_env_dir}/bin/easy_install -U distribute'.format(**kwargs) - print(cmd) - os.system(cmd) - - for package in get_reqs(test=True, pv=float(pv)): - kwargs['package'] = package - cmd = '{virtual_env_dir}/bin/pip install -U {package}'.format(**kwargs) - print(cmd) - os.system(cmd) - - def run(self): - versions = self.versions - if self.pv: - versions = [self.pv] - - for pv in versions: - - self.build_virtualenv(pv) - kwargs = dict( - pv=pv, - virtual_env_dir=self.virtual_env_dir % pv, - name=self.name) - - if self.name: - cmd = '{virtual_env_dir}/bin/django-admin.py test --pythonpath=. --settings=database_files.tests.settings database_files.tests.tests.{name}'.format(**kwargs) - else: - cmd = '{virtual_env_dir}/bin/django-admin.py test --pythonpath=. --settings=database_files.tests.settings database_files.tests'.format(**kwargs) +# def get_reqs(test=False, pv=None): +# reqs = [ +# 'Django>=1.4', +# 'six>=1.7.2', +# ] +# if test: +# #TODO:remove once Django 1.7 south integration becomes main-stream? +# if not pv: +# reqs.append('South>=1.0') +# elif pv <= 3.2: +# # Note, South dropped Python3 support after 0.8.4... +# reqs.append('South==0.8.4') +# return reqs - print(cmd) - ret = os.system(cmd) - if ret: - return +# class TestCommand(Command): +# description = "Runs unittests." +# user_options = [ +# ('name=', None, +# 'Name of the specific test to run.'), +# ('virtual-env-dir=', None, +# 'The location of the virtual environment to use.'), +# ('pv=', None, +# 'The version of Python to use. e.g. 2.7 or 3'), +# ] +# +# def initialize_options(self): +# self.name = None +# self.virtual_env_dir = '.env%s' +# self.pv = 0 +# self.versions = [ +# 2.7, +# 3, +# #3.3,#TODO? +# ] +# +# def finalize_options(self): +# pass +# +# def build_virtualenv(self, pv): +# virtual_env_dir = self.virtual_env_dir % pv +# kwargs = dict(virtual_env_dir=virtual_env_dir, pv=pv) +# if not os.path.isdir(virtual_env_dir): +# cmd = 'virtualenv -p /usr/bin/python{pv} {virtual_env_dir}'.format(**kwargs) +# print(cmd) +# os.system(cmd) +# +# cmd = '{virtual_env_dir}/bin/easy_install -U distribute'.format(**kwargs) +# print(cmd) +# os.system(cmd) +# +# for package in get_reqs(test=True, pv=float(pv)): +# kwargs['package'] = package +# cmd = '{virtual_env_dir}/bin/pip install -U {package}'.format(**kwargs) +# print(cmd) +# os.system(cmd) +# +# def run(self): +# versions = self.versions +# if self.pv: +# versions = [self.pv] +# +# for pv in versions: +# +# self.build_virtualenv(pv) +# kwargs = dict( +# pv=pv, +# virtual_env_dir=self.virtual_env_dir % pv, +# name=self.name) +# +# if self.name: +# cmd = '{virtual_env_dir}/bin/django-admin.py test --pythonpath=. --settings=database_files.tests.settings database_files.tests.tests.{name}'.format(**kwargs) +# else: +# cmd = '{virtual_env_dir}/bin/django-admin.py test --pythonpath=. --settings=database_files.tests.settings database_files.tests'.format(**kwargs) +# +# print(cmd) +# ret = os.system(cmd) +# if ret: +# return try: long_description = read_md('README.md') @@ -124,8 +136,6 @@ def run(self): 'Programming Language :: Python :: 3.0', 'Programming Language :: Python :: 3.2', ], - install_requires = get_reqs(), - cmdclass={ - 'test': TestCommand, - }, + install_requires=get_reqs('pip-requirements-min-django.txt', 'pip-requirements.txt'), + tests_require=get_reqs('pip-requirements-test.txt'), ) diff --git a/tox.ini b/tox.ini new file mode 100644 index 0000000..d9ec4da --- /dev/null +++ b/tox.ini @@ -0,0 +1,32 @@ +[tox] +envlist = py{27,34,35}-django{15,16,17,18,19,110} +recreate = True + +[testenv] +basepython = + py27: python2.7 + py33: python3.3 + py34: python3.4 + py35: python3.5 +deps = + -r{toxinidir}/pip-requirements.txt + -r{toxinidir}/pip-requirements-test.txt + django15: Django>=1.5,<1.6 + django16: Django>=1.6,<1.7 + django17: Django>=1.7,<1.8 + django18: Django>=1.8,<1.9 + django19: Django>=1.9,<1.10 + django110: Django>=1.10,<2 +commands = django-admin.py test --traceback --pythonpath=. --settings=database_files.tests.settings database_files.tests.tests.DatabaseFilesTestCase{env:TESTNAME:} + +# Django 1.5 uses a different test module lookup mechanism, so it needs a different command. +[testenv:py27-django15] +commands = django-admin.py test --traceback --pythonpath=. --settings=database_files.tests.settings tests.DatabaseFilesTestCase{env:TESTNAME:} +[testenv:py34-django15] +commands = django-admin.py test --traceback --pythonpath=. --settings=database_files.tests.settings tests.DatabaseFilesTestCase{env:TESTNAME:} +[testenv:py35-django15] +commands = django-admin.py test --traceback --pythonpath=. --settings=database_files.tests.settings tests.DatabaseFilesTestCase{env:TESTNAME:} + +# TODO: fix, PID times-out in Travis-CI? +#[testenv:py34-django19] +#commands = python --version From 328691c1f80ad50041abe9fbf3b59a215aeb0eb1 Mon Sep 17 00:00:00 2001 From: chrisspen Date: Tue, 11 Oct 2016 18:49:29 -0400 Subject: [PATCH 46/82] Updated docs. --- README.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/README.md b/README.md index 5b95064..f081f71 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,8 @@ Django Database Files 3000 ========================== +![Build Status](https://secure.travis-ci.org/chrisspen/django-database-files-3000.png?branch=master) + This is a storage system for Django that stores uploaded files in the database. Files can be served from the database (usually a bad idea), the file system, or a CDN. From 07f919011eec1b1820ea835f62e4bd8c14a9652d Mon Sep 17 00:00:00 2001 From: ngenhit Date: Sat, 15 Oct 2016 00:15:55 +0530 Subject: [PATCH 47/82] the bytestring was breaking the migration. If there is no particular reason to have a bytestring in place of the original package's string db column, then we can revert to the string itself. --- database_files/migrations/0001_initial.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/database_files/migrations/0001_initial.py b/database_files/migrations/0001_initial.py index 1eda234..59b3c87 100644 --- a/database_files/migrations/0001_initial.py +++ b/database_files/migrations/0001_initial.py @@ -20,9 +20,9 @@ class Migration(migrations.Migration): ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), ('name', models.CharField(db_index=True, max_length=255, unique=True)), ('size', models.PositiveIntegerField(db_index=True)), - ('_content', models.TextField(db_column=b'content')), - ('created_datetime', models.DateTimeField(db_index=True, default=django.utils.timezone.now, verbose_name=b'Created datetime')), - ('_content_hash', models.CharField(blank=True, db_column=b'content_hash', db_index=True, max_length=128, null=True)), + ('_content', models.TextField(db_column='content')), + ('created_datetime', models.DateTimeField(db_index=True, default=django.utils.timezone.now, verbose_name='Created datetime')), + ('_content_hash', models.CharField(blank=True, db_column='content_hash', db_index=True, max_length=128, null=True)), ], options={ 'db_table': 'database_files_file', From e5f78840357351974888fde00f4dbf3e7aca80ac Mon Sep 17 00:00:00 2001 From: chrisspen Date: Sat, 15 Oct 2016 14:06:31 -0400 Subject: [PATCH 48/82] changed travis link --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index f081f71..c75e028 100644 --- a/README.md +++ b/README.md @@ -1,7 +1,7 @@ Django Database Files 3000 ========================== -![Build Status](https://secure.travis-ci.org/chrisspen/django-database-files-3000.png?branch=master) +[Build Status](https://travis-ci.org/chrisspen/django-database-files-3000) This is a storage system for Django that stores uploaded files in the database. Files can be served from the database From 0ecf0f7bf13c256b6d6d3a147e2c65d717ee36dd Mon Sep 17 00:00:00 2001 From: chrisspen Date: Sat, 15 Oct 2016 14:16:54 -0400 Subject: [PATCH 49/82] fixed unittests for django>=1.7 --- database_files/tests/settings.py | 19 ++++++++++++++++++- 1 file changed, 18 insertions(+), 1 deletion(-) diff --git a/database_files/tests/settings.py b/database_files/tests/settings.py index 185510e..4625bcd 100644 --- a/database_files/tests/settings.py +++ b/database_files/tests/settings.py @@ -20,7 +20,6 @@ 'django.contrib.sites', 'database_files', 'database_files.tests', - 'south', ] DEFAULT_FILE_STORAGE = 'database_files.storage.DatabaseStorage' @@ -43,3 +42,21 @@ def __getitem__(self, item): USE_TZ = True SECRET_KEY = 'secret' + +AUTH_USER_MODEL = 'auth.User' + +SITE_ID = 1 + +BASE_SECURE_URL = 'https://localhost' + +BASE_URL = 'http://localhost' + +MIDDLEWARE_CLASSES = ( + 'django.middleware.common.CommonMiddleware', + 'django.contrib.sessions.middleware.SessionMiddleware', + 'django.middleware.csrf.CsrfViewMiddleware', + #'django.middleware.transaction.TransactionMiddleware', + 'django.contrib.auth.middleware.AuthenticationMiddleware', + 'django.contrib.messages.middleware.MessageMiddleware', + 'django.middleware.locale.LocaleMiddleware', +) From b5376b41c7f67ed714c87ced750db50c587c6f32 Mon Sep 17 00:00:00 2001 From: chrisspen Date: Sat, 15 Oct 2016 15:32:07 -0400 Subject: [PATCH 50/82] Fixed unittests for all supported Python+Django versions. --- database_files/__init__.py | 2 +- database_files/tests/settings.py | 3 --- database_files/tests/tests.py | 10 +++++++--- database_files/urls.py | 29 +++++++++++++++++++++++------ tox.ini | 21 +++++++++++++-------- 5 files changed, 44 insertions(+), 21 deletions(-) diff --git a/database_files/__init__.py b/database_files/__init__.py index c6d9b06..48f40db 100644 --- a/database_files/__init__.py +++ b/database_files/__init__.py @@ -1,2 +1,2 @@ -VERSION = (1, 0, 0) +VERSION = (1, 0, 1) __version__ = '.'.join(map(str, VERSION)) \ No newline at end of file diff --git a/database_files/tests/settings.py b/database_files/tests/settings.py index 4625bcd..a916fc1 100644 --- a/database_files/tests/settings.py +++ b/database_files/tests/settings.py @@ -5,9 +5,6 @@ DATABASES = { 'default':{ 'ENGINE': 'django.db.backends.sqlite3', - # Don't do this. It dramatically slows down the test. -# 'NAME': '/tmp/database_files.db', -# 'TEST_NAME': '/tmp/database_files.db', } } diff --git a/database_files/tests/tests.py b/database_files/tests/tests.py index 4631744..4b03d98 100644 --- a/database_files/tests/tests.py +++ b/database_files/tests/tests.py @@ -9,6 +9,7 @@ from django.test import TestCase from django.core.files.storage import default_storage from django.core.management import call_command +from django.conf import settings from database_files.models import File from database_files.tests.models import Thing @@ -61,9 +62,12 @@ def test_adding_file(self): # Load a dynamically created file outside /media. test_file = files.temp.NamedTemporaryFile( suffix='.txt', - dir=files.temp.gettempdir() + # Django>=1.10 no longer allows accessing files outside of MEDIA_ROOT... + #dir=files.temp.gettempdir() + dir=os.path.join(settings.PROJECT_DIR, 'media'), ) - test_file.write(b'1234567890') + data0 = b'1234567890' + test_file.write(data0) test_file.seek(0) t = Thing.objects.create( upload=files.File(test_file), @@ -72,7 +76,7 @@ def test_adding_file(self): t = Thing.objects.get(pk=t.pk) self.assertEqual(t.upload.file.size, 10) self.assertEqual(t.upload.file.name[-4:], '.txt') - self.assertEqual(t.upload.file.read(), b'1234567890') + self.assertEqual(t.upload.file.read(), data0) t.upload.delete() self.assertEqual(File.objects.count(), 1) diff --git a/database_files/urls.py b/database_files/urls.py index 4f508a9..1721896 100644 --- a/database_files/urls.py +++ b/database_files/urls.py @@ -1,15 +1,32 @@ try: # Removed in Django 1.6 - from django.conf.urls.defaults import patterns, url + from django.conf.urls.defaults import url except ImportError: - from django.conf.urls import patterns, url + from django.conf.urls import url + +try: + # Relocated in Django 1.6 + from django.conf.urls.defaults import pattern +except ImportError: + # Completely removed in Django 1.10 + try: + from django.conf.urls import patterns + except ImportError: + patterns = None -urlpatterns = patterns('', +import database_files.views + +_patterns = [ # url(r'^files/(?P.+)$', -# 'database_files.views.serve', +# database_files.views.serve, # name='database_file'), url(r'^files/(?P.+)$', - 'database_files.views.serve_mixed', + database_files.views.serve_mixed, name='database_file'), -) +] + +if patterns is None: + urlpatterns = _patterns +else: + urlpatterns = patterns('', *_patterns) diff --git a/tox.ini b/tox.ini index d9ec4da..012f1bc 100644 --- a/tox.ini +++ b/tox.ini @@ -1,10 +1,19 @@ [tox] -envlist = py{27,34,35}-django{15,16,17,18,19,110} +#https://pypi.python.org/pypi/Django/1.5 +#https://pypi.python.org/pypi/Django/1.6 +#https://pypi.python.org/pypi/Django/1.7 +#https://pypi.python.org/pypi/Django/1.8 +#https://pypi.python.org/pypi/Django/1.9 +#https://pypi.python.org/pypi/Django/1.10 +# Note, several versions support Python 3.2, but Pip has dropped support, so we can't test them. +# See https://github.com/travis-ci/travis-ci/issues/5485 +envlist = py{27,33}-django{15,16},py{27,33,34}-django{17,18},py{27,34,35}-django{19},py{27,34,35}-django{110} recreate = True [testenv] basepython = py27: python2.7 + py32: python3.2 py33: python3.3 py34: python3.4 py35: python3.5 @@ -22,11 +31,7 @@ commands = django-admin.py test --traceback --pythonpath=. --settings=database_f # Django 1.5 uses a different test module lookup mechanism, so it needs a different command. [testenv:py27-django15] commands = django-admin.py test --traceback --pythonpath=. --settings=database_files.tests.settings tests.DatabaseFilesTestCase{env:TESTNAME:} -[testenv:py34-django15] +#[testenv:py32-django15] +#commands = django-admin.py test --traceback --pythonpath=. --settings=database_files.tests.settings tests.DatabaseFilesTestCase{env:TESTNAME:} +[testenv:py33-django15] commands = django-admin.py test --traceback --pythonpath=. --settings=database_files.tests.settings tests.DatabaseFilesTestCase{env:TESTNAME:} -[testenv:py35-django15] -commands = django-admin.py test --traceback --pythonpath=. --settings=database_files.tests.settings tests.DatabaseFilesTestCase{env:TESTNAME:} - -# TODO: fix, PID times-out in Travis-CI? -#[testenv:py34-django19] -#commands = python --version From 737bab630f56bbf9220af8026f498b05c06fe7df Mon Sep 17 00:00:00 2001 From: chrisspen Date: Sat, 15 Oct 2016 22:33:01 -0400 Subject: [PATCH 51/82] Fixed code formatting. --- .travis.yml | 3 +- database_files/__init__.py | 2 +- .../commands/database_files_cleanup.py | 8 +- .../commands/database_files_load.py | 19 +- database_files/manager.py | 4 +- database_files/models.py | 11 +- database_files/tests/settings.py | 3 +- database_files/tests/tests.py | 12 +- database_files/utils.py | 38 +- database_files/views.py | 3 +- pep8.sh | 2 + pylint.messages | 750 ++++++++++++++++++ pylint.rc | 356 +++++++++ setup.py | 96 +-- 14 files changed, 1166 insertions(+), 141 deletions(-) create mode 100755 pep8.sh create mode 100644 pylint.messages create mode 100644 pylint.rc diff --git a/.travis.yml b/.travis.yml index 216627b..4b861ac 100644 --- a/.travis.yml +++ b/.travis.yml @@ -2,6 +2,7 @@ language: python python: - "3.5" install: -- pip install tox +- pip install tox pylint script: +- ./pep8.sh - tox diff --git a/database_files/__init__.py b/database_files/__init__.py index 48f40db..e3114ec 100644 --- a/database_files/__init__.py +++ b/database_files/__init__.py @@ -1,2 +1,2 @@ VERSION = (1, 0, 1) -__version__ = '.'.join(map(str, VERSION)) \ No newline at end of file +__version__ = '.'.join(map(str, VERSION)) diff --git a/database_files/management/commands/database_files_cleanup.py b/database_files/management/commands/database_files_cleanup.py index 5b9d532..178c630 100644 --- a/database_files/management/commands/database_files_cleanup.py +++ b/database_files/management/commands/database_files_cleanup.py @@ -52,12 +52,12 @@ def handle(self, *args, **options): subq_i += 1 if subq_i == 1 or not subq_i % 100: print('%i of %i' % (subq_i, subq_total)) - file = getattr(row, field.name) - if file is None: + f = getattr(row, field.name) + if f is None: continue - if not file.name: + if not f.name: continue - names.add(file.name) + names.add(f.name) # Find all database files with names not in our list. print('Finding orphaned files...') diff --git a/database_files/management/commands/database_files_load.py b/database_files/management/commands/database_files_load.py index b28b5a6..835e321 100644 --- a/database_files/management/commands/database_files_load.py +++ b/database_files/management/commands/database_files_load.py @@ -1,14 +1,13 @@ from __future__ import print_function import os +from optparse import make_option from django.conf import settings from django.core.files.storage import default_storage from django.core.management.base import BaseCommand, CommandError from django.db.models import FileField, ImageField, get_models -from optparse import make_option - class Command(BaseCommand): option_list = BaseCommand.option_list + ( make_option('-m', '--models', @@ -31,7 +30,7 @@ def handle(self, *args, **options): try: broken = 0 # Number of db records referencing missing files. for model in get_models(): - key = "%s.%s" % (model._meta.app_label,model._meta.module_name) + key = "%s.%s" % (model._meta.app_label, model._meta.module_name) key = key.lower() if all_models and key not in all_models: continue @@ -45,19 +44,19 @@ def handle(self, *args, **options): xq = {field.name:''} for row in model.objects.filter(**q).exclude(**xq): try: - file = getattr(row, field.name) - if file is None: + f = getattr(row, field.name) + if f is None: continue - if not file.name: + if not f.name: continue if show_files: - print("\t",file.name) - if file.path and not os.path.isfile(file.path): + print("\t", f.name) + if f.path and not os.path.isfile(f.path): if show_files: - print("Broken:",file.name) + print("Broken:", f.name) broken += 1 continue - file.read() + f.read() row.save() except IOError: broken += 1 diff --git a/database_files/manager.py b/database_files/manager.py index 3ce79f5..11d15f9 100644 --- a/database_files/manager.py +++ b/database_files/manager.py @@ -1,7 +1,7 @@ -from django.db import models import os +from django.db import models + class FileManager(models.Manager): def get_from_name(self, name): return self.get(name=name) - \ No newline at end of file diff --git a/database_files/models.py b/database_files/models.py index c833072..78ad27e 100644 --- a/database_files/models.py +++ b/database_files/models.py @@ -4,8 +4,6 @@ import six -from . import settings as _settings - from django.conf import settings from django.db import models from django.utils import timezone @@ -14,6 +12,8 @@ from database_files.utils import write_file, is_fresh from database_files.manager import FileManager +from . import settings as _settings + class File(models.Model): objects = FileManager() @@ -115,14 +115,13 @@ def dump_files(cls, debug=True, verbose=False): if verbose: print(('File %i-%s is stale. Writing to local file ' 'system...') % (file_id, name)) - file = File.objects.get(id=file_id) + f = File.objects.get(id=file_id) write_file( file.name, file.content, overwrite=True) - file._content_hash = None - file.save() + f._content_hash = None + f.save() finally: if debug: settings.DEBUG = tmp_debug - \ No newline at end of file diff --git a/database_files/tests/settings.py b/database_files/tests/settings.py index a916fc1..1abd6db 100644 --- a/database_files/tests/settings.py +++ b/database_files/tests/settings.py @@ -1,4 +1,5 @@ -import os, sys +import os +import sys PROJECT_DIR = os.path.dirname(__file__) diff --git a/database_files/tests/tests.py b/database_files/tests/tests.py index 4b03d98..229091a 100644 --- a/database_files/tests/tests.py +++ b/database_files/tests/tests.py @@ -30,12 +30,12 @@ def test_adding_file(self): # Create default thing storing reference to file # in the local media directory. test_fqfn = os.path.join(self.media_dir, 'test.txt') - open(test_fqfn,'w').write('hello there') + open(test_fqfn, 'w').write('hello there') o1 = o = Thing() test_fn = 'i/special/test.txt' o.upload = test_fn o.save() - id = o.id + obj_id = o.id # Confirm thing was saved. Thing.objects.update() @@ -49,7 +49,7 @@ def test_adding_file(self): self.assertEqual(q.count(), 0) # Verify we can read the contents of thing. - o = Thing.objects.get(id=id) + o = Thing.objects.get(id=obj_id) self.assertEqual(o.upload.read(), b"hello there") # Verify that by attempting to read the file, we've automatically @@ -106,7 +106,7 @@ def test_hash(self): open(fqfn, 'wb').write(image_content) # Calculate hash from various sources and confirm they all match. - expected_hash = '35830221efe45ab0dc3d91ca23c29d2d3c20d00c9afeaa096ab256ec322a7a0b3293f07a01377e31060e65b4e5f6f8fdb4c0e56bc586bba5a7ab3e6d6d97a192' + expected_hash = '35830221efe45ab0dc3d91ca23c29d2d3c20d00c9afeaa096ab256ec322a7a0b3293f07a01377e31060e65b4e5f6f8fdb4c0e56bc586bba5a7ab3e6d6d97a192' # pylint: disable=C0301 h = utils.get_text_hash(image_content) self.assertEqual(h, expected_hash) h = utils.get_file_hash(fqfn) @@ -124,15 +124,13 @@ def test_hash(self): fqfn = os.path.join(self.media_dir, 'test.txt') open(fqfn, 'wb').write(image_content.encode('utf-8')) - expected_hash = '1f40fc92da241694750979ee6cf582f2d5d7d28e18335de05abc54d0560e0f5302860c652bf08d560252aa5e74210546f369fbbbce8c12cfc7957b2652fe9a75' + expected_hash = '1f40fc92da241694750979ee6cf582f2d5d7d28e18335de05abc54d0560e0f5302860c652bf08d560252aa5e74210546f369fbbbce8c12cfc7957b2652fe9a75' # pylint: disable=C0301 h = utils.get_text_hash(image_content) self.assertEqual(h, expected_hash) h = utils.get_file_hash(fqfn) self.assertEqual(h, expected_hash) h = utils.get_text_hash(open(fqfn, 'rb').read()) self.assertEqual(h, expected_hash) -# h = utils.get_text_hash(open(fqfn, 'r').read()) -# self.assertEqual(h, expected_hash) def test_reading_file(self): call_command('loaddata', 'test_files.json') diff --git a/database_files/utils.py b/database_files/utils.py index b43ea15..41ba845 100644 --- a/database_files/utils.py +++ b/database_files/utils.py @@ -1,5 +1,4 @@ -#from grp import getgrnam -#from pwd import getpwnam + import os import hashlib @@ -40,7 +39,7 @@ def get_hash_fn(name): """ fqfn = os.path.join(settings.MEDIA_ROOT, name) fqfn = os.path.normpath(fqfn) - dirs,fn = os.path.split(fqfn) + dirs, fn = os.path.split(fqfn) if not os.path.isdir(dirs): os.makedirs(dirs) fqfn_parts = os.path.split(fqfn) @@ -56,19 +55,19 @@ def write_file(name, content, overwrite=False): fqfn = os.path.normpath(fqfn) if os.path.isfile(fqfn) and not overwrite: return - dirs,fn = os.path.split(fqfn) + dirs, fn = os.path.split(fqfn) if not os.path.isdir(dirs): os.makedirs(dirs) open(fqfn, 'wb').write(content) # Cache hash. - hash = get_file_hash(fqfn) + hash_value = get_file_hash(fqfn) hash_fn = get_hash_fn(name) try: # Write out bytes in Python3. - value = six.binary_type(hash, 'utf-8') + value = six.binary_type(hash_value, 'utf-8') except TypeError: - value = hash + value = hash_value open(hash_fn, 'wb').write(value) # Set ownership and permissions. @@ -84,14 +83,17 @@ def write_file(name, content, overwrite=False): if perms: os.system('chmod -R %s "%s"' % (perms, dirs)) -def get_file_hash(fin, - force_encoding=settings.DB_FILES_DEFAULT_ENFORCE_ENCODING, - encoding=settings.DB_FILES_DEFAULT_ENCODING, - errors=settings.DB_FILES_DEFAULT_ERROR_METHOD, - chunk_size=128): +def get_file_hash(fin, force_encoding=None, encoding=None, errors=None, chunk_size=128): """ Iteratively builds a file hash without loading the entire file into memory. """ + + force_encoding = force_encoding or settings.DB_FILES_DEFAULT_ENFORCE_ENCODING + + encoding = encoding or settings.DB_FILES_DEFAULT_ENCODING + + errors = errors or settings.DB_FILES_DEFAULT_ERROR_METHOD + if isinstance(fin, six.string_types): fin = open(fin, 'rb') h = hashlib.sha512() @@ -117,13 +119,17 @@ def get_text_hash_0004(text): h.update(text.encode('utf-8', 'replace')) return h.hexdigest() -def get_text_hash(text, - force_encoding=settings.DB_FILES_DEFAULT_ENFORCE_ENCODING, - encoding=settings.DB_FILES_DEFAULT_ENCODING, - errors=settings.DB_FILES_DEFAULT_ERROR_METHOD): +def get_text_hash(text, force_encoding=None, encoding=None, errors=None): """ Returns the hash of the given text. """ + + force_encoding = force_encoding or settings.DB_FILES_DEFAULT_ENFORCE_ENCODING + + encoding = encoding or settings.DB_FILES_DEFAULT_ENCODING + + errors = errors or settings.DB_FILES_DEFAULT_ERROR_METHOD + h = hashlib.sha512() if force_encoding: if not isinstance(text, six.text_type): diff --git a/database_files/views.py b/database_files/views.py index 0c9dd83..5dd1f05 100644 --- a/database_files/views.py +++ b/database_files/views.py @@ -1,4 +1,5 @@ import base64 +import mimetypes import os from django.conf import settings @@ -7,8 +8,6 @@ from django.views.decorators.cache import cache_control from django.views.static import serve as django_serve -import mimetypes - from database_files.models import File @cache_control(max_age=86400) diff --git a/pep8.sh b/pep8.sh new file mode 100755 index 0000000..f4fe378 --- /dev/null +++ b/pep8.sh @@ -0,0 +1,2 @@ +#!/bin/bash +pylint --rcfile=pylint.rc database_files diff --git a/pylint.messages b/pylint.messages new file mode 100644 index 0000000..f856863 --- /dev/null +++ b/pylint.messages @@ -0,0 +1,750 @@ +:blacklisted-name (C0102): *Black listed name "%s"* + Used when the name is listed in the black list (unauthorized names). +:invalid-name (C0103): *Invalid %s name "%s"%s* + Used when the name doesn't match the regular expression associated to its type + (constant, variable, class...). +:missing-docstring (C0111): *Missing %s docstring* + Used when a module, function, class or method has no docstring.Some special + methods like __init__ doesn't necessary require a docstring. +:empty-docstring (C0112): *Empty %s docstring* + Used when a module, function, class or method has an empty docstring (it would + be too easy ;). +:unneeded-not (C0113): *Consider changing "%s" to "%s"* + Used when a boolean expression contains an unneeded negation. +:singleton-comparison (C0121): *Comparison to %s should be %s* + Used when an expression is compared to singleton values like True, False or + None. +:misplaced-comparison-constant (C0122): *Comparison should be %s* + Used when the constant is placed on the left sideof a comparison. It is + usually clearer in intent to place it in the right hand side of the + comparison. +:unidiomatic-typecheck (C0123): *Using type() instead of isinstance() for a typecheck.* + The idiomatic way to perform an explicit typecheck in Python is to use + isinstance(x, Y) rather than type(x) == Y, type(x) is Y. Though there are + unusual situations where these give different results. +:consider-using-enumerate (C0200): *Consider using enumerate instead of iterating with range and len* + Emitted when code that iterates with range and len is encountered. Such code + can be simplified by using the enumerate builtin. +:consider-iterating-dictionary (C0201): *Consider iterating the dictionary directly instead of calling .keys()* + Emitted when the keys of a dictionary are iterated through the .keys() method. + It is enough to just iterate through the dictionary itself, as in "for key in + dictionary". +:bad-classmethod-argument (C0202): *Class method %s should have %s as first argument* + Used when a class method has a first argument named differently than the value + specified in valid-classmethod-first-arg option (default to "cls"), + recommended to easily differentiate them from regular instance methods. +:bad-mcs-method-argument (C0203): *Metaclass method %s should have %s as first argument* + Used when a metaclass method has a first agument named differently than the + value specified in valid-classmethod-first-arg option (default to "cls"), + recommended to easily differentiate them from regular instance methods. +:bad-mcs-classmethod-argument (C0204): *Metaclass class method %s should have %s as first argument* + Used when a metaclass class method has a first argument named differently than + the value specified in valid-metaclass-classmethod-first-arg option (default + to "mcs"), recommended to easily differentiate them from regular instance + methods. +:line-too-long (C0301): *Line too long (%s/%s)* + Used when a line is longer than a given number of characters. +:too-many-lines (C0302): *Too many lines in module (%s/%s)* + Used when a module has too much lines, reducing its readability. +:trailing-whitespace (C0303): *Trailing whitespace* + Used when there is whitespace between the end of a line and the newline. +:missing-final-newline (C0304): *Final newline missing* + Used when the last line in a file is missing a newline. +:trailing-newlines (C0305): *Trailing newlines* + Used when there are trailing blank lines in a file. +:multiple-statements (C0321): *More than one statement on a single line* + Used when more than on statement are found on the same line. +:superfluous-parens (C0325): *Unnecessary parens after %r keyword* + Used when a single item in parentheses follows an if, for, or other keyword. +:bad-whitespace (C0326): *%s space %s %s %s* + Used when a wrong number of spaces is used around an operator, bracket or + block opener. +:mixed-line-endings (C0327): *Mixed line endings LF and CRLF* + Used when there are mixed (LF and CRLF) newline signs in a file. +:unexpected-line-ending-format (C0328): *Unexpected line ending format. There is '%s' while it should be '%s'.* + Used when there is different newline than expected. +:bad-continuation (C0330): *Wrong %s indentation%s%s.* + TODO +:wrong-spelling-in-comment (C0401): *Wrong spelling of a word '%s' in a comment:* + Used when a word in comment is not spelled correctly. +:wrong-spelling-in-docstring (C0402): *Wrong spelling of a word '%s' in a docstring:* + Used when a word in docstring is not spelled correctly. +:invalid-characters-in-docstring (C0403): *Invalid characters %r in a docstring* + Used when a word in docstring cannot be checked by enchant. +:multiple-imports (C0410): *Multiple imports on one line (%s)* + Used when import statement importing multiple modules is detected. +:wrong-import-order (C0411): *%s comes before %s* + Used when PEP8 import order is not respected (standard imports first, then + third-party libraries, then local imports) +:ungrouped-imports (C0412): *Imports from package %s are not grouped* + Used when imports are not grouped by packages +:wrong-import-position (C0413): *Import "%s" should be placed at the top of the module* + Used when code and imports are mixed +:old-style-class (C1001): *Old-style class defined.* + Used when a class is defined that does not inherit from anotherclass and does + not inherit explicitly from "object". This message can't be emitted when using + Python >= 3.0. +:syntax-error (E0001): + Used when a syntax error is raised for a module. +:unrecognized-inline-option (E0011): *Unrecognized file option %r* + Used when an unknown inline option is encountered. +:bad-option-value (E0012): *Bad option value %r* + Used when a bad value for an inline option is encountered. +:init-is-generator (E0100): *__init__ method is a generator* + Used when the special class method __init__ is turned into a generator by a + yield in its body. +:return-in-init (E0101): *Explicit return in __init__* + Used when the special class method __init__ has an explicit return value. +:function-redefined (E0102): *%s already defined line %s* + Used when a function / class / method is redefined. +:not-in-loop (E0103): *%r not properly in loop* + Used when break or continue keywords are used outside a loop. +:return-outside-function (E0104): *Return outside function* + Used when a "return" statement is found outside a function or method. +:yield-outside-function (E0105): *Yield outside function* + Used when a "yield" statement is found outside a function or method. +:return-arg-in-generator (E0106): *Return with argument inside generator* + Used when a "return" statement with an argument is found outside in a + generator function or method (e.g. with some "yield" statements). This message + can't be emitted when using Python >= 3.3. +:nonexistent-operator (E0107): *Use of the non-existent %s operator* + Used when you attempt to use the C-style pre-increment orpre-decrement + operator -- and ++, which doesn't exist in Python. +:duplicate-argument-name (E0108): *Duplicate argument name %s in function definition* + Duplicate argument names in function definitions are syntax errors. +:abstract-class-instantiated (E0110): *Abstract class %r with abstract methods instantiated* + Used when an abstract class with `abc.ABCMeta` as metaclass has abstract + methods and is instantiated. +:bad-reversed-sequence (E0111): *The first reversed() argument is not a sequence* + Used when the first argument to reversed() builtin isn't a sequence (does not + implement __reversed__, nor __getitem__ and __len__ +:continue-in-finally (E0116): *'continue' not supported inside 'finally' clause* + Emitted when the `continue` keyword is found inside a finally clause, which is + a SyntaxError. +:method-hidden (E0202): *An attribute defined in %s line %s hides this method* + Used when a class defines a method which is hidden by an instance attribute + from an ancestor class or set by some client code. +:access-member-before-definition (E0203): *Access to member %r before its definition line %s* + Used when an instance member is accessed before it's actually assigned. +:no-method-argument (E0211): *Method has no argument* + Used when a method which should have the bound instance as first argument has + no argument defined. +:no-self-argument (E0213): *Method should have "self" as first argument* + Used when a method has an attribute different the "self" as first argument. + This is considered as an error since this is a so common convention that you + shouldn't break it! +:invalid-slots-object (E0236): *Invalid object %r in __slots__, must contain only non empty strings* + Used when an invalid (non-string) object occurs in __slots__. +:assigning-non-slot (E0237): *Assigning to attribute %r not defined in class slots* + Used when assigning to an attribute not defined in the class slots. +:invalid-slots (E0238): *Invalid __slots__ object* + Used when an invalid __slots__ is found in class. Only a string, an iterable + or a sequence is permitted. +:inherit-non-class (E0239): *Inheriting %r, which is not a class.* + Used when a class inherits from something which is not a class. +:inconsistent-mro (E0240): *Inconsistent method resolution order for class %r* + Used when a class has an inconsistent method resolutin order. +:duplicate-bases (E0241): *Duplicate bases for class %r* + Used when a class has duplicate bases. +:non-iterator-returned (E0301): *__iter__ returns non-iterator* + Used when an __iter__ method returns something which is not an iterable (i.e. + has no `next` method) +:unexpected-special-method-signature (E0302): *The special method %r expects %s param(s), %d %s given* + Emitted when a special method was defined with an invalid number of + parameters. If it has too few or too many, it might not work at all. +:invalid-length-returned (E0303): *__len__ does not return non-negative integer* + Used when an __len__ method returns something which is not a non-negative + integer +:import-error (E0401): *Unable to import %s* + Used when pylint has been unable to import a module. +:used-before-assignment (E0601): *Using variable %r before assignment* + Used when a local variable is accessed before it's assignment. +:undefined-variable (E0602): *Undefined variable %r* + Used when an undefined variable is accessed. +:undefined-all-variable (E0603): *Undefined variable name %r in __all__* + Used when an undefined variable name is referenced in __all__. +:invalid-all-object (E0604): *Invalid object %r in __all__, must contain only strings* + Used when an invalid (non-string) object occurs in __all__. +:no-name-in-module (E0611): *No name %r in module %r* + Used when a name cannot be found in a module. +:unbalanced-tuple-unpacking (E0632): *Possible unbalanced tuple unpacking with sequence%s: left side has %d label(s), right side has %d value(s)* + Used when there is an unbalanced tuple unpacking in assignment +:unpacking-non-sequence (E0633): *Attempting to unpack a non-sequence%s* + Used when something which is not a sequence is used in an unpack assignment +:bad-except-order (E0701): *Bad except clauses order (%s)* + Used when except clauses are not in the correct order (from the more specific + to the more generic). If you don't fix the order, some exceptions may not be + catched by the most specific handler. +:raising-bad-type (E0702): *Raising %s while only classes or instances are allowed* + Used when something which is neither a class, an instance or a string is + raised (i.e. a `TypeError` will be raised). +:misplaced-bare-raise (E0704): *The raise statement is not inside an except clause* + Used when a bare raise is not used inside an except clause. This generates an + error, since there are no active exceptions to be reraised. An exception to + this rule is represented by a bare raise inside a finally clause, which might + work, as long as an exception is raised inside the try block, but it is + nevertheless a code smell that must not be relied upon. +:raising-non-exception (E0710): *Raising a new style class which doesn't inherit from BaseException* + Used when a new style class which doesn't inherit from BaseException is + raised. +:notimplemented-raised (E0711): *NotImplemented raised - should raise NotImplementedError* + Used when NotImplemented is raised instead of NotImplementedError +:catching-non-exception (E0712): *Catching an exception which doesn't inherit from BaseException: %s* + Used when a class which doesn't inherit from BaseException is used as an + exception in an except clause. +:slots-on-old-class (E1001): *Use of __slots__ on an old style class* + Used when an old style class uses the __slots__ attribute. This message can't + be emitted when using Python >= 3.0. +:super-on-old-class (E1002): *Use of super on an old style class* + Used when an old style class uses the super builtin. This message can't be + emitted when using Python >= 3.0. +:bad-super-call (E1003): *Bad first argument %r given to super()* + Used when another argument than the current class is given as first argument + of the super builtin. +:missing-super-argument (E1004): *Missing argument to super()* + Used when the super builtin didn't receive an argument. This message can't be + emitted when using Python >= 3.0. +:no-member (E1101): *%s %r has no %r member* + Used when a variable is accessed for an unexistent member. +:not-callable (E1102): *%s is not callable* + Used when an object being called has been inferred to a non callable object +:assignment-from-no-return (E1111): *Assigning to function call which doesn't return* + Used when an assignment is done on a function call but the inferred function + doesn't return anything. +:no-value-for-parameter (E1120): *No value for argument %s in %s call* + Used when a function call passes too few arguments. +:too-many-function-args (E1121): *Too many positional arguments for %s call* + Used when a function call passes too many positional arguments. +:unexpected-keyword-arg (E1123): *Unexpected keyword argument %r in %s call* + Used when a function call passes a keyword argument that doesn't correspond to + one of the function's parameter names. +:redundant-keyword-arg (E1124): *Argument %r passed by position and keyword in %s call* + Used when a function call would result in assigning multiple values to a + function parameter, one value from a positional argument and one from a + keyword argument. +:invalid-sequence-index (E1126): *Sequence index is not an int, slice, or instance with __index__* + Used when a sequence type is indexed with an invalid type. Valid types are + ints, slices, and objects with an __index__ method. +:invalid-slice-index (E1127): *Slice index is not an int, None, or instance with __index__* + Used when a slice index is not an integer, None, or an object with an + __index__ method. +:assignment-from-none (E1128): *Assigning to function call which only returns None* + Used when an assignment is done on a function call but the inferred function + returns nothing but None. +:not-context-manager (E1129): *Context manager '%s' doesn't implement __enter__ and __exit__.* + Used when an instance in a with statement doesn't implement the context + manager protocol(__enter__/__exit__). +:invalid-unary-operand-type (E1130): + Emitted when an unary operand is used on an object which does not support this + type of operation +:unsupported-binary-operation (E1131): + Emitted when a binary arithmetic operation between two operands is not + supported. +:repeated-keyword (E1132): *Got multiple values for keyword argument %r in function call* + Emitted when a function call got multiple values for a keyword. +:not-an-iterable (E1133): *Non-iterable value %s is used in an iterating context* + Used when a non-iterable value is used in place whereiterable is expected +:not-a-mapping (E1134): *Non-mapping value %s is used in a mapping context* + Used when a non-mapping value is used in place wheremapping is expected +:unsupported-membership-test (E1135): *Value '%s' doesn't support membership test* + Emitted when an instance in membership test expression doesn'timplement + membership protocol (__contains__/__iter__/__getitem__) +:unsubscriptable-object (E1136): *Value '%s' is unsubscriptable* + Emitted when a subscripted value doesn't support subscription(i.e. doesn't + define __getitem__ method) +:logging-unsupported-format (E1200): *Unsupported logging format character %r (%#02x) at index %d* + Used when an unsupported format character is used in a logging statement + format string. +:logging-format-truncated (E1201): *Logging format string ends in middle of conversion specifier* + Used when a logging statement format string terminates before the end of a + conversion specifier. +:logging-too-many-args (E1205): *Too many arguments for logging format string* + Used when a logging format string is given too few arguments. +:logging-too-few-args (E1206): *Not enough arguments for logging format string* + Used when a logging format string is given too many arguments +:bad-format-character (E1300): *Unsupported format character %r (%#02x) at index %d* + Used when a unsupported format character is used in a format string. +:truncated-format-string (E1301): *Format string ends in middle of conversion specifier* + Used when a format string terminates before the end of a conversion specifier. +:mixed-format-string (E1302): *Mixing named and unnamed conversion specifiers in format string* + Used when a format string contains both named (e.g. '%(foo)d') and unnamed + (e.g. '%d') conversion specifiers. This is also used when a named conversion + specifier contains * for the minimum field width and/or precision. +:format-needs-mapping (E1303): *Expected mapping for format string, not %s* + Used when a format string that uses named conversion specifiers is used with + an argument that is not a mapping. +:missing-format-string-key (E1304): *Missing key %r in format string dictionary* + Used when a format string that uses named conversion specifiers is used with a + dictionary that doesn't contain all the keys required by the format string. +:too-many-format-args (E1305): *Too many arguments for format string* + Used when a format string that uses unnamed conversion specifiers is given too + many arguments. +:too-few-format-args (E1306): *Not enough arguments for format string* + Used when a format string that uses unnamed conversion specifiers is given too + few arguments +:bad-str-strip-call (E1310): *Suspicious argument in %s.%s call* + The argument to a str.{l,r,}strip call contains a duplicate character, +:print-statement (E1601): *print statement used* + Used when a print statement is used (`print` is a function in Python 3) This + message can't be emitted when using Python >= 3.0. +:parameter-unpacking (E1602): *Parameter unpacking specified* + Used when parameter unpacking is specified for a function(Python 3 doesn't + allow it) This message can't be emitted when using Python >= 3.0. +:unpacking-in-except (E1603): *Implicit unpacking of exceptions is not supported in Python 3* + Python3 will not allow implicit unpacking of exceptions in except clauses. See + http://www.python.org/dev/peps/pep-3110/ This message can't be emitted when + using Python >= 3.0. +:old-raise-syntax (E1604): *Use raise ErrorClass(args) instead of raise ErrorClass, args.* + Used when the alternate raise syntax 'raise foo, bar' is used instead of + 'raise foo(bar)'. This message can't be emitted when using Python >= 3.0. +:backtick (E1605): *Use of the `` operator* + Used when the deprecated "``" (backtick) operator is used instead of the str() + function. This message can't be emitted when using Python >= 3.0. +:long-suffix (E1606): *Use of long suffix* + Used when "l" or "L" is used to mark a long integer. This will not work in + Python 3, since `int` and `long` types have merged. This message can't be + emitted when using Python >= 3.0. +:old-ne-operator (E1607): *Use of the <> operator* + Used when the deprecated "<>" operator is used instead of "!=". This is + removed in Python 3. This message can't be emitted when using Python >= 3.0. +:old-octal-literal (E1608): *Use of old octal literal* + Usen when encountering the old octal syntax, removed in Python 3. To use the + new syntax, prepend 0o on the number. This message can't be emitted when using + Python >= 3.0. +:import-star-module-level (E1609): *Import * only allowed at module level* + Used when the import star syntax is used somewhere else than the module level. + This message can't be emitted when using Python >= 3.0. +:fatal (F0001): + Used when an error occurred preventing the analysis of a module (unable to + find it for instance). +:astroid-error (F0002): *%s: %s* + Used when an unexpected error occurred while building the Astroid + representation. This is usually accompanied by a traceback. Please report such + errors ! +:parse-error (F0010): *error while code parsing: %s* + Used when an exception occured while building the Astroid representation which + could be handled by astroid. +:method-check-failed (F0202): *Unable to check methods signature (%s / %s)* + Used when Pylint has been unable to check methods signature compatibility for + an unexpected reason. Please report this kind if you don't make sense of it. +:raw-checker-failed (I0001): *Unable to run raw checkers on built-in module %s* + Used to inform that a built-in module has not been checked using the raw + checkers. +:bad-inline-option (I0010): *Unable to consider inline option %r* + Used when an inline option is either badly formatted or can't be used inside + modules. +:locally-disabled (I0011): *Locally disabling %s (%s)* + Used when an inline option disables a message or a messages category. +:locally-enabled (I0012): *Locally enabling %s (%s)* + Used when an inline option enables a message or a messages category. +:file-ignored (I0013): *Ignoring entire file* + Used to inform that the file will not be checked +:suppressed-message (I0020): *Suppressed %s (from line %d)* + A message was triggered on a line, but suppressed explicitly by a disable= + comment in the file. This message is not generated for messages that are + ignored due to configuration settings. +:useless-suppression (I0021): *Useless suppression of %s* + Reported when a message is explicitly disabled for a line or a block of code, + but never triggered. +:deprecated-pragma (I0022): *Pragma "%s" is deprecated, use "%s" instead* + Some inline pylint options have been renamed or reworked, only the most recent + form should be used. NOTE:skip-all is only available with pylint >= 0.26 +:too-many-nested-blocks (R0101): *Too many nested blocks (%s/%s)* + Used when a function or a method has too many nested blocks. This makes the + code less understandable and maintainable. +:simplifiable-if-statement (R0102): *The if statement can be replaced with %s* + Used when an if statement can be replaced with 'bool(test)'. +:no-self-use (R0201): *Method could be a function* + Used when a method doesn't use its bound instance, and so could be written as + a function. +:no-classmethod-decorator (R0202): *Consider using a decorator instead of calling classmethod* + Used when a class method is defined without using the decorator syntax. +:no-staticmethod-decorator (R0203): *Consider using a decorator instead of calling staticmethod* + Used when a static method is defined without using the decorator syntax. +:redefined-variable-type (R0204): *Redefinition of %s type from %s to %s* + Used when the type of a variable changes inside a method or a function. +:cyclic-import (R0401): *Cyclic import (%s)* + Used when a cyclic import between two or more modules is detected. +:duplicate-code (R0801): *Similar lines in %s files* + Indicates that a set of similar lines has been detected among multiple file. + This usually means that the code should be refactored to avoid this + duplication. +:too-many-ancestors (R0901): *Too many ancestors (%s/%s)* + Used when class has too many parent classes, try to reduce this to get a + simpler (and so easier to use) class. +:too-many-instance-attributes (R0902): *Too many instance attributes (%s/%s)* + Used when class has too many instance attributes, try to reduce this to get a + simpler (and so easier to use) class. +:too-few-public-methods (R0903): *Too few public methods (%s/%s)* + Used when class has too few public methods, so be sure it's really worth it. +:too-many-public-methods (R0904): *Too many public methods (%s/%s)* + Used when class has too many public methods, try to reduce this to get a + simpler (and so easier to use) class. +:too-many-return-statements (R0911): *Too many return statements (%s/%s)* + Used when a function or method has too many return statement, making it hard + to follow. +:too-many-branches (R0912): *Too many branches (%s/%s)* + Used when a function or method has too many branches, making it hard to + follow. +:too-many-arguments (R0913): *Too many arguments (%s/%s)* + Used when a function or method takes too many arguments. +:too-many-locals (R0914): *Too many local variables (%s/%s)* + Used when a function or method has too many local variables. +:too-many-statements (R0915): *Too many statements (%s/%s)* + Used when a function or method has too many statements. You should then split + it in smaller functions / methods. +:too-many-boolean-expressions (R0916): *Too many boolean expressions in if statement (%s/%s)* + Used when a if statement contains too many boolean expressions +:unreachable (W0101): *Unreachable code* + Used when there is some code behind a "return" or "raise" statement, which + will never be accessed. +:dangerous-default-value (W0102): *Dangerous default value %s as argument* + Used when a mutable value as list or dictionary is detected in a default value + for an argument. +:pointless-statement (W0104): *Statement seems to have no effect* + Used when a statement doesn't have (or at least seems to) any effect. +:pointless-string-statement (W0105): *String statement has no effect* + Used when a string is used as a statement (which of course has no effect). + This is a particular case of W0104 with its own message so you can easily + disable it if you're using those strings as documentation, instead of + comments. +:expression-not-assigned (W0106): *Expression "%s" is assigned to nothing* + Used when an expression that is not a function call is assigned to nothing. + Probably something else was intended. +:unnecessary-pass (W0107): *Unnecessary pass statement* + Used when a "pass" statement that can be avoided is encountered. +:unnecessary-lambda (W0108): *Lambda may not be necessary* + Used when the body of a lambda expression is a function call on the same + argument list as the lambda itself; such lambda expressions are in all but a + few cases replaceable with the function being called in the body of the + lambda. +:duplicate-key (W0109): *Duplicate key %r in dictionary* + Used when a dictionary expression binds the same key multiple times. +:deprecated-lambda (W0110): *map/filter on lambda could be replaced by comprehension* + Used when a lambda is the first argument to "map" or "filter". It could be + clearer as a list comprehension or generator expression. This message can't be + emitted when using Python >= 3.0. +:useless-else-on-loop (W0120): *Else clause on loop without a break statement* + Loops should only have an else clause if they can exit early with a break + statement, otherwise the statements under else should be on the same scope as + the loop itself. +:exec-used (W0122): *Use of exec* + Used when you use the "exec" statement (function for Python 3), to discourage + its usage. That doesn't mean you can not use it ! +:eval-used (W0123): *Use of eval* + Used when you use the "eval" function, to discourage its usage. Consider using + `ast.literal_eval` for safely evaluating strings containing Python expressions + from untrusted sources. +:confusing-with-statement (W0124): *Following "as" with another context manager looks like a tuple.* + Emitted when a `with` statement component returns multiple values and uses + name binding with `as` only for a part of those values, as in with ctx() as a, + b. This can be misleading, since it's not clear if the context manager returns + a tuple or if the node without a name binding is another context manager. +:using-constant-test (W0125): *Using a conditional statement with a constant value* + Emitted when a conditional statement (If or ternary if) uses a constant value + for its test. This might not be what the user intended to do. +:lost-exception (W0150): *%s statement in finally block may swallow exception* + Used when a break or a return statement is found inside the finally clause of + a try...finally block: the exceptions raised in the try clause will be + silently swallowed instead of being re-raised. +:assert-on-tuple (W0199): *Assert called on a 2-uple. Did you mean 'assert x,y'?* + A call of assert on a tuple will always evaluate to true if the tuple is not + empty, and will always evaluate to false if it is. +:attribute-defined-outside-init (W0201): *Attribute %r defined outside __init__* + Used when an instance attribute is defined outside the __init__ method. +:bad-staticmethod-argument (W0211): *Static method with %r as first argument* + Used when a static method has "self" or a value specified in valid- + classmethod-first-arg option or valid-metaclass-classmethod-first-arg option + as first argument. +:protected-access (W0212): *Access to a protected member %s of a client class* + Used when a protected member (i.e. class member with a name beginning with an + underscore) is access outside the class or a descendant of the class where + it's defined. +:arguments-differ (W0221): *Arguments number differs from %s %r method* + Used when a method has a different number of arguments than in the implemented + interface or in an overridden method. +:signature-differs (W0222): *Signature differs from %s %r method* + Used when a method signature is different than in the implemented interface or + in an overridden method. +:abstract-method (W0223): *Method %r is abstract in class %r but is not overridden* + Used when an abstract method (i.e. raise NotImplementedError) is not + overridden in concrete class. +:super-init-not-called (W0231): *__init__ method from base class %r is not called* + Used when an ancestor class method has an __init__ method which is not called + by a derived class. +:no-init (W0232): *Class has no __init__ method* + Used when a class has no __init__ method, neither its parent classes. +:non-parent-init-called (W0233): *__init__ method from a non direct base class %r is called* + Used when an __init__ method is called on a class which is not in the direct + ancestors for the analysed class. +:unnecessary-semicolon (W0301): *Unnecessary semicolon* + Used when a statement is ended by a semi-colon (";"), which isn't necessary + (that's python, not C ;). +:bad-indentation (W0311): *Bad indentation. Found %s %s, expected %s* + Used when an unexpected number of indentation's tabulations or spaces has been + found. +:mixed-indentation (W0312): *Found indentation with %ss instead of %ss* + Used when there are some mixed tabs and spaces in a module. +:lowercase-l-suffix (W0332): *Use of "l" as long integer identifier* + Used when a lower case "l" is used to mark a long integer. You should use a + upper case "L" since the letter "l" looks too much like the digit "1" This + message can't be emitted when using Python >= 3.0. +:wildcard-import (W0401): *Wildcard import %s* + Used when `from module import *` is detected. +:deprecated-module (W0402): *Uses of a deprecated module %r* + Used a module marked as deprecated is imported. +:relative-import (W0403): *Relative import %r, should be %r* + Used when an import relative to the package directory is detected. This + message can't be emitted when using Python >= 3.0. +:reimported (W0404): *Reimport %r (imported line %s)* + Used when a module is reimported multiple times. +:import-self (W0406): *Module import itself* + Used when a module is importing itself. +:misplaced-future (W0410): *__future__ import is not the first non docstring statement* + Python 2.5 and greater require __future__ import to be the first non docstring + statement in the module. +:fixme (W0511): + Used when a warning note as FIXME or XXX is detected. +:invalid-encoded-data (W0512): *Cannot decode using encoding "%s", unexpected byte at position %d* + Used when a source line cannot be decoded using the specified source file + encoding. This message can't be emitted when using Python >= 3.0. +:global-variable-undefined (W0601): *Global variable %r undefined at the module level* + Used when a variable is defined through the "global" statement but the + variable is not defined in the module scope. +:global-variable-not-assigned (W0602): *Using global for %r but no assignment is done* + Used when a variable is defined through the "global" statement but no + assignment to this variable is done. +:global-statement (W0603): *Using the global statement* + Used when you use the "global" statement to update a global variable. Pylint + just try to discourage this usage. That doesn't mean you can not use it ! +:global-at-module-level (W0604): *Using the global statement at the module level* + Used when you use the "global" statement at the module level since it has no + effect +:unused-import (W0611): *Unused %s* + Used when an imported module or variable is not used. +:unused-variable (W0612): *Unused variable %r* + Used when a variable is defined but not used. +:unused-argument (W0613): *Unused argument %r* + Used when a function or method argument is not used. +:unused-wildcard-import (W0614): *Unused import %s from wildcard import* + Used when an imported module or variable is not used from a `'from X import + *'` style import. +:redefined-outer-name (W0621): *Redefining name %r from outer scope (line %s)* + Used when a variable's name hide a name defined in the outer scope. +:redefined-builtin (W0622): *Redefining built-in %r* + Used when a variable or function override a built-in. +:redefine-in-handler (W0623): *Redefining name %r from %s in exception handler* + Used when an exception handler assigns the exception to an existing name +:undefined-loop-variable (W0631): *Using possibly undefined loop variable %r* + Used when an loop variable (i.e. defined by a for loop or a list comprehension + or a generator expression) is used outside the loop. +:cell-var-from-loop (W0640): *Cell variable %s defined in loop* + A variable used in a closure is defined in a loop. This will result in all + closures using the same value for the closed-over variable. +:bare-except (W0702): *No exception type(s) specified* + Used when an except clause doesn't specify exceptions type to catch. +:broad-except (W0703): *Catching too general exception %s* + Used when an except catches a too general exception, possibly burying + unrelated errors. +:duplicate-except (W0705): *Catching previously caught exception type %s* + Used when an except catches a type that was already caught by a previous + handler. +:nonstandard-exception (W0710): *Exception doesn't inherit from standard "Exception" class* + Used when a custom exception class is raised but doesn't inherit from the + builtin "Exception" class. This message can't be emitted when using Python >= + 3.0. +:binary-op-exception (W0711): *Exception to catch is the result of a binary "%s" operation* + Used when the exception to catch is of the form "except A or B:". If intending + to catch multiple, rewrite as "except (A, B):" +:property-on-old-class (W1001): *Use of "property" on an old style class* + Used when Pylint detect the use of the builtin "property" on an old style + class while this is relying on new style classes features. This message can't + be emitted when using Python >= 3.0. +:logging-not-lazy (W1201): *Specify string format arguments as logging function parameters* + Used when a logging statement has a call form of "logging.(format_string % (format_args...))". Such calls should leave string + interpolation to the logging method itself and be written "logging.(format_string, format_args...)" so that the program may avoid + incurring the cost of the interpolation in those cases in which no message + will be logged. For more, see http://www.python.org/dev/peps/pep-0282/. +:logging-format-interpolation (W1202): *Use % formatting in logging functions and pass the % parameters as arguments* + Used when a logging statement has a call form of "logging.(format_string.format(format_args...))". Such calls should use % + formatting instead, but leave interpolation to the logging function by passing + the parameters as arguments. +:bad-format-string-key (W1300): *Format string dictionary key should be a string, not %s* + Used when a format string that uses named conversion specifiers is used with a + dictionary whose keys are not all strings. +:unused-format-string-key (W1301): *Unused key %r in format string dictionary* + Used when a format string that uses named conversion specifiers is used with a + dictionary that contains keys not required by the format string. +:bad-format-string (W1302): *Invalid format string* + Used when a PEP 3101 format string is invalid. This message can't be emitted + when using Python < 2.7. +:missing-format-argument-key (W1303): *Missing keyword argument %r for format string* + Used when a PEP 3101 format string that uses named fields doesn't receive one + or more required keywords. This message can't be emitted when using Python < + 2.7. +:unused-format-string-argument (W1304): *Unused format argument %r* + Used when a PEP 3101 format string that uses named fields is used with an + argument that is not required by the format string. This message can't be + emitted when using Python < 2.7. +:format-combined-specification (W1305): *Format string contains both automatic field numbering and manual field specification* + Usen when a PEP 3101 format string contains both automatic field numbering + (e.g. '{}') and manual field specification (e.g. '{0}'). This message can't be + emitted when using Python < 2.7. +:missing-format-attribute (W1306): *Missing format attribute %r in format specifier %r* + Used when a PEP 3101 format string uses an attribute specifier ({0.length}), + but the argument passed for formatting doesn't have that attribute. This + message can't be emitted when using Python < 2.7. +:invalid-format-index (W1307): *Using invalid lookup key %r in format specifier %r* + Used when a PEP 3101 format string uses a lookup specifier ({a[1]}), but the + argument passed for formatting doesn't contain or doesn't have that key as an + attribute. This message can't be emitted when using Python < 2.7. +:anomalous-backslash-in-string (W1401): *Anomalous backslash in string: '%s'. String constant might be missing an r prefix.* + Used when a backslash is in a literal string but not as an escape. +:anomalous-unicode-escape-in-string (W1402): *Anomalous Unicode escape in byte string: '%s'. String constant might be missing an r or u prefix.* + Used when an escape like \u is encountered in a byte string where it has no + effect. +:bad-open-mode (W1501): *"%s" is not a valid mode for open.* + Python supports: r, w, a[, x] modes with b, +, and U (only with r) options. + See http://docs.python.org/2/library/functions.html#open +:boolean-datetime (W1502): *Using datetime.time in a boolean context.* + Using datetime.time in a boolean context can hide subtle bugs when the time + they represent matches midnight UTC. This behaviour was fixed in Python 3.5. + See http://bugs.python.org/issue13936 for reference. This message can't be + emitted when using Python >= 3.5. +:redundant-unittest-assert (W1503): *Redundant use of %s with constant value %r* + The first argument of assertTrue and assertFalse is a condition. If a constant + is passed as parameter, that condition will be always true. In this case a + warning should be emitted. +:deprecated-method (W1505): *Using deprecated method %s()* + The method is marked as deprecated and will be removed in a future version of + Python. Consider looking for an alternative in the documentation. +:apply-builtin (W1601): *apply built-in referenced* + Used when the apply built-in function is referenced (missing from Python 3) + This message can't be emitted when using Python >= 3.0. +:basestring-builtin (W1602): *basestring built-in referenced* + Used when the basestring built-in function is referenced (missing from Python + 3) This message can't be emitted when using Python >= 3.0. +:buffer-builtin (W1603): *buffer built-in referenced* + Used when the buffer built-in function is referenced (missing from Python 3) + This message can't be emitted when using Python >= 3.0. +:cmp-builtin (W1604): *cmp built-in referenced* + Used when the cmp built-in function is referenced (missing from Python 3) This + message can't be emitted when using Python >= 3.0. +:coerce-builtin (W1605): *coerce built-in referenced* + Used when the coerce built-in function is referenced (missing from Python 3) + This message can't be emitted when using Python >= 3.0. +:execfile-builtin (W1606): *execfile built-in referenced* + Used when the execfile built-in function is referenced (missing from Python 3) + This message can't be emitted when using Python >= 3.0. +:file-builtin (W1607): *file built-in referenced* + Used when the file built-in function is referenced (missing from Python 3) + This message can't be emitted when using Python >= 3.0. +:long-builtin (W1608): *long built-in referenced* + Used when the long built-in function is referenced (missing from Python 3) + This message can't be emitted when using Python >= 3.0. +:raw_input-builtin (W1609): *raw_input built-in referenced* + Used when the raw_input built-in function is referenced (missing from Python + 3) This message can't be emitted when using Python >= 3.0. +:reduce-builtin (W1610): *reduce built-in referenced* + Used when the reduce built-in function is referenced (missing from Python 3) + This message can't be emitted when using Python >= 3.0. +:standarderror-builtin (W1611): *StandardError built-in referenced* + Used when the StandardError built-in function is referenced (missing from + Python 3) This message can't be emitted when using Python >= 3.0. +:unicode-builtin (W1612): *unicode built-in referenced* + Used when the unicode built-in function is referenced (missing from Python 3) + This message can't be emitted when using Python >= 3.0. +:xrange-builtin (W1613): *xrange built-in referenced* + Used when the xrange built-in function is referenced (missing from Python 3) + This message can't be emitted when using Python >= 3.0. +:coerce-method (W1614): *__coerce__ method defined* + Used when a __coerce__ method is defined (method is not used by Python 3) This + message can't be emitted when using Python >= 3.0. +:delslice-method (W1615): *__delslice__ method defined* + Used when a __delslice__ method is defined (method is not used by Python 3) + This message can't be emitted when using Python >= 3.0. +:getslice-method (W1616): *__getslice__ method defined* + Used when a __getslice__ method is defined (method is not used by Python 3) + This message can't be emitted when using Python >= 3.0. +:setslice-method (W1617): *__setslice__ method defined* + Used when a __setslice__ method is defined (method is not used by Python 3) + This message can't be emitted when using Python >= 3.0. +:no-absolute-import (W1618): *import missing `from __future__ import absolute_import`* + Used when an import is not accompanied by ``from __future__ import + absolute_import`` (default behaviour in Python 3) This message can't be + emitted when using Python >= 3.0. +:old-division (W1619): *division w/o __future__ statement* + Used for non-floor division w/o a float literal or ``from __future__ import + division`` (Python 3 returns a float for int division unconditionally) This + message can't be emitted when using Python >= 3.0. +:dict-iter-method (W1620): *Calling a dict.iter*() method* + Used for calls to dict.iterkeys(), itervalues() or iteritems() (Python 3 lacks + these methods) This message can't be emitted when using Python >= 3.0. +:dict-view-method (W1621): *Calling a dict.view*() method* + Used for calls to dict.viewkeys(), viewvalues() or viewitems() (Python 3 lacks + these methods) This message can't be emitted when using Python >= 3.0. +:next-method-called (W1622): *Called a next() method on an object* + Used when an object's next() method is called (Python 3 uses the next() built- + in function) This message can't be emitted when using Python >= 3.0. +:metaclass-assignment (W1623): *Assigning to a class's __metaclass__ attribute* + Used when a metaclass is specified by assigning to __metaclass__ (Python 3 + specifies the metaclass as a class statement argument) This message can't be + emitted when using Python >= 3.0. +:indexing-exception (W1624): *Indexing exceptions will not work on Python 3* + Indexing exceptions will not work on Python 3. Use `exception.args[index]` + instead. This message can't be emitted when using Python >= 3.0. +:raising-string (W1625): *Raising a string exception* + Used when a string exception is raised. This will not work on Python 3. This + message can't be emitted when using Python >= 3.0. +:reload-builtin (W1626): *reload built-in referenced* + Used when the reload built-in function is referenced (missing from Python 3). + You can use instead imp.reload or importlib.reload. This message can't be + emitted when using Python >= 3.0. +:oct-method (W1627): *__oct__ method defined* + Used when a __oct__ method is defined (method is not used by Python 3) This + message can't be emitted when using Python >= 3.0. +:hex-method (W1628): *__hex__ method defined* + Used when a __hex__ method is defined (method is not used by Python 3) This + message can't be emitted when using Python >= 3.0. +:nonzero-method (W1629): *__nonzero__ method defined* + Used when a __nonzero__ method is defined (method is not used by Python 3) + This message can't be emitted when using Python >= 3.0. +:cmp-method (W1630): *__cmp__ method defined* + Used when a __cmp__ method is defined (method is not used by Python 3) This + message can't be emitted when using Python >= 3.0. +:input-builtin (W1632): *input built-in referenced* + Used when the input built-in is referenced (backwards-incompatible semantics + in Python 3) This message can't be emitted when using Python >= 3.0. +:round-builtin (W1633): *round built-in referenced* + Used when the round built-in is referenced (backwards-incompatible semantics + in Python 3) This message can't be emitted when using Python >= 3.0. +:intern-builtin (W1634): *intern built-in referenced* + Used when the intern built-in is referenced (Moved to sys.intern in Python 3) + This message can't be emitted when using Python >= 3.0. +:unichr-builtin (W1635): *unichr built-in referenced* + Used when the unichr built-in is referenced (Use chr in Python 3) This message + can't be emitted when using Python >= 3.0. +:map-builtin-not-iterating (W1636): *map built-in referenced when not iterating* + Used when the map built-in is referenced in a non-iterating context (returns + an iterator in Python 3) This message can't be emitted when using Python >= + 3.0. +:zip-builtin-not-iterating (W1637): *zip built-in referenced when not iterating* + Used when the zip built-in is referenced in a non-iterating context (returns + an iterator in Python 3) This message can't be emitted when using Python >= + 3.0. +:range-builtin-not-iterating (W1638): *range built-in referenced when not iterating* + Used when the range built-in is referenced in a non-iterating context (returns + an iterator in Python 3) This message can't be emitted when using Python >= + 3.0. +:filter-builtin-not-iterating (W1639): *filter built-in referenced when not iterating* + Used when the filter built-in is referenced in a non-iterating context + (returns an iterator in Python 3) This message can't be emitted when using + Python >= 3.0. +:using-cmp-argument (W1640): *Using the cmp argument for list.sort / sorted* + Using the cmp argument for list.sort or the sorted builtin should be avoided, + since it was removed in Python 3. Using either `key` or `functools.cmp_to_key` + should be preferred. This message can't be emitted when using Python >= 3.0. + diff --git a/pylint.rc b/pylint.rc new file mode 100644 index 0000000..dce15c4 --- /dev/null +++ b/pylint.rc @@ -0,0 +1,356 @@ +# lint Python modules using external checkers. +# +# This is the main checker controlling the other ones and the reports +# generation. It is itself both a raw checker and an astng checker in order +# to: +# * handle message activation / deactivation at the module level +# * handle some basic but necessary stats'data (number of classes, methods...) +# +[MASTER] + +# Specify a configuration file. +#rcfile= + +# Python code to execute, usually for sys.path manipulation such as +# pygtk.require(). +#init-hook= + +# Profiled execution. +profile=no + +# Add to the black list. It should be a base name, not a +# path. You may set this option multiple times. +# Ignore all auto-generated South migration directories. +ignore=migrations,south_migrations + +# Pickle collected data for later comparisons. +persistent=yes + +# Set the cache size for astng objects. +cache-size=500 + +# List of plugins (as comma separated values of python modules names) to load, +# usually to register additional checkers. +load-plugins= + +[MESSAGES CONTROL] + +# Enable only checker(s) with the given id(s). This option conflicts with the +# disable-checker option +#enable-checker= + +# Enable all checker(s) except those with the given id(s). This option +# conflicts with the enable-checker option +#disable-checker= + +# Enable all messages in the listed categories (IRCWEF). +#enable-msg-cat= + +# Disable all messages in the listed categories (IRCWEF). +disable-msg-cat=I + +# Enable the message(s) with the given id(s). +#enable-msg= + +#http://docs.pylint.org/features.html +#http://pylint-messages.wikidot.com/all-codes +#pylint --list-msgs > pylint.messages + +# All these are disabled below. +# C1001: old-style class defined (Django uses these for Meta options) +# C0103: variable regex check. +# C0111: missing docstring check. It's too vague. Complains about no docstrings in __init__ and other places we don't care about. +# C0303: Trailing whitespace. +# C0330: bad-continuation +# E1101: member check...this is usually wrong. +# E1103: type inference...this is usually wrong. +# F0401: unable to import +# R0201: method should be function check. +# R0401: cyclic import check...because sometimes it's wrong. +# R0902: too many instance attributes check. +# R0903: too few public methods check...makes no sense with Django. +# R0904: too many public method check. +# R0913: too many argument check. +# R0921: abstract class not referenced check. +# W0104: no effect check. +# W0142: magic check. +# W0212: protected data check. +# W0232: __init__ check. +# W0311: bad-indentation +# W0401: wildcard import. +# W0404: reimport check...this is sometimes wrong. +# W0511: TODO check. +# W0612: unused variable check. +# W0613: unused argument check. Too vague. +# W0614: wildcard import usage check. +# W0704: empty except check. +# E1002: Use of super on an old style class +# E1120: No value for argument +# R0901: Too many ancestors +# W0611: Unused import +# E1123: Unexpected keyword argument %r in %s call +# C0302: *Too many lines in module (%s)* +# R0801: *Similar lines in %s files* +# R0914: *Too many local variables (%s/%s)* +# R0912: *Too many branches (%s/%s)* +# R0915: *Too many statements (%s/%s)* +# W0703: *Catching too general exception %s* +# E1003: *Bad first argument %r given to super()* +# E0202: *An attribute defined in %s line %s hides this method* +# W0201: *Attribute %r defined outside __init__* +# W0221: *Arguments number differs from %s method* +# C0325: *Unnecessary parens after %r keyword* +# R0916: too-many-boolean-expressions +# R0204: *Redefinition of %s type from %s to %s* +# R0101: *Too many nested blocks (%s/%s)* +# I0011: *Locally disabling %s (%s)* +disable=C1001,C0103,R0201,W0212,W0614,W0401,W0704,E1101,W0142,R0904,R0913,W0404,R0903,W0232,C0111,W0613,W0612,W0511,W0104,R0902,R0921,R0401,E1103,C0303,W0311,C0330,F0401,E1002,E1120,R0901,W0611,E1123,C0302,R0801,R0914,R0912,R0915,W0703,E1003,E0202,W0201,W0221,C0325,R0916,R0204,R0101,I0011 + +[REPORTS] + +# Set the output format. Available formats are text, parseable, colorized, msvs +# (visual studio) and html +output-format=text + +# Include message's id in output +include-ids=yes + +# Put messages in a separate file for each module / package specified on the +# command line instead of printing them on stdout. Reports (if any) will be +# written in a file name "pylint_global.[txt|html]". +files-output=no + +# Tells whether to display a full report or only the messages +reports=yes + +# Python expression which should return a note less than 10 (10 is the highest +# note). You have access to the variables errors warning, statement which +# respectively contain the number of errors / warnings messages and the total +# number of statements analyzed. This is used by the global evaluation report +# (R0004). +evaluation=10.0 - ((float(5 * error + warning + refactor + convention) / statement) * 10) + +# Add a comment according to your evaluation note. This is used by the global +# evaluation report (R0004). +comment=no + +# Enable the report(s) with the given id(s). +#enable-report= + +# Disable the report(s) with the given id(s). +#disable-report= + + +# checks for : +# * doc strings +# * modules / classes / functions / methods / arguments / variables name +# * number of arguments, local variables, branches, returns and statements in +# functions, methods +# * required module attributes +# * dangerous default values as arguments +# * redefinition of function / method / class +# * uses of the global statement +# +[BASIC] + +# Regular expression which should only match functions or classes name which do +# not require a docstring +no-docstring-rgx=__.*__ + +# Regular expression which should only match correct module names +module-rgx=(([a-z_][a-z0-9_]*)|([A-Z][a-zA-Z0-9]+))$ + +# Regular expression which should only match correct module level names +const-rgx=(([A-Z_][A-Z0-9_]*)|(__.*__))$ + +# Regular expression which should only match correct class names +class-rgx=[A-Z_][a-zA-Z0-9]+$ + +# Regular expression which should only match correct function names +function-rgx=[a-z_][a-z0-9_]{2,30}$ + +# Regular expression which should only match correct method names +method-rgx=[a-z_][a-z0-9_]{2,30}$ + +# Regular expression which should only match correct instance attribute names +attr-rgx=[a-z_][a-z0-9_]{2,30}$ + +# Regular expression which should only match correct argument names +argument-rgx=[a-z_][a-z0-9_]{2,30}$ + +# Regular expression which should only match correct variable names +variable-rgx=[a-z_][a-z0-9_]{2,30}$ + +# Regular expression which should only match correct list comprehension / +# generator expression variable names +inlinevar-rgx=[A-Za-z_][A-Za-z0-9_]*$ + +# Good variable names which should always be accepted, separated by a comma +good-names=i,j,k,ex,Run,_ + +# Bad variable names which should always be refused, separated by a comma +bad-names=foo,bar,baz,toto,tutu,tata + +# List of builtins function names that should not be used, separated by a comma +bad-functions=map,filter,apply,input + + +# try to find bugs in the code using type inference +# +[TYPECHECK] + +# Tells whether missing members accessed in mixin class should be ignored. A +# mixin class is detected if its name ends with "mixin" (case insensitive). +ignore-mixin-members=yes + +# List of classes names for which member attributes should not be checked +# (useful for classes with attributes dynamically set). +ignored-classes=SQLObject + +# When zope mode is activated, add a predefined set of Zope acquired attributes +# to generated-members. +zope=no + +# List of members which are set dynamically and missed by pylint inference +# system, and so shouldn't trigger E0201 when accessed. +generated-members=REQUEST,acl_users,aq_parent + + +# checks for +# * unused variables / imports +# * undefined variables +# * redefinition of variable from builtins or from an outer scope +# * use of variable before assignment +# +[VARIABLES] + +# Tells whether we should check for unused import in __init__ files. +init-import=no + +# A regular expression matching names used for dummy variables (i.e. not used). +dummy-variables-rgx=_|dummy + +# List of additional names supposed to be defined in builtins. Remember that +# you should avoid to define new builtins when possible. +additional-builtins= + + +# checks for +# * external modules dependencies +# * relative / wildcard imports +# * cyclic imports +# * uses of deprecated modules +# +[IMPORTS] + +# Deprecated modules which should not be used, separated by a comma +deprecated-modules=regsub,string,TERMIOS,Bastion,rexec + +# Create a graph of every (i.e. internal and external) dependencies in the +# given file (report R0402 must not be disabled) +import-graph= + +# Create a graph of external dependencies in the given file (report R0402 must +# not be disabled) +ext-import-graph= + +# Create a graph of internal dependencies in the given file (report R0402 must +# not be disabled) +int-import-graph= + + +# checks for sign of poor/misdesign: +# * number of methods, attributes, local variables... +# * size, complexity of functions, methods +# +[DESIGN] + +# Maximum number of arguments for function / method +max-args=5 + +# Maximum number of locals for function / method body +max-locals=15 + +# Maximum number of return / yield for function / method body +max-returns=6 + +# Maximum number of branch for function / method body +max-branchs=12 + +# Maximum number of statements in function / method body +max-statements=50 + +# Maximum number of parents for a class (see R0901). +max-parents=7 + +# Maximum number of attributes for a class (see R0902). +max-attributes=7 + +# Minimum number of public methods for a class (see R0903). +min-public-methods=2 + +# Maximum number of public methods for a class (see R0904). +max-public-methods=20 + + +# checks for : +# * methods without self as first argument +# * overridden methods signature +# * access only to existent members via self +# * attributes not defined in the __init__ method +# * supported interfaces implementation +# * unreachable code +# +[CLASSES] + +# List of interface methods to ignore, separated by a comma. This is used for +# instance to not check methods defines in Zope's Interface base class. +#ignore-iface-methods=isImplementedBy,deferred,extends,names,namesAndDescriptions,queryDescriptionFor,getBases,getDescriptionFor,getDoc,getName,getTaggedValue,getTaggedValueTags,isEqualOrExtendedBy,setTaggedValue,isImplementedByInstancesOf,adaptWith,is_implemented_by + +# List of method names used to declare (i.e. assign) instance attributes. +defining-attr-methods=__init__,__new__,setUp + + +# checks for similarities and duplicated code. This computation may be +# memory / CPU intensive, so you should disable it if you experiments some +# problems. +# +[SIMILARITIES] + +# Minimum lines number of a similarity. +min-similarity-lines=4 + +# Ignore comments when computing similarities. +ignore-comments=yes + +# Ignore docstrings when computing similarities. +ignore-docstrings=yes + + +# checks for : +# * unauthorized constructions +# * strict indentation +# * line length +# * use of <> instead of != +# +[FORMAT] + +# Maximum number of characters on a single line. +max-line-length=100 + +# Maximum number of lines in a module +max-module-lines=1000 + +# String used as indentation unit. This is usually " " (4 spaces) or "\t" (1 +# tab). +indent-string=' ' + + +# checks for: +# * warning notes in the code like FIXME, XXX +# * PEP 263: source code with non ascii character but no encoding declaration +# +[MISCELLANEOUS] + +# List of note tags to take in consideration, separated by a comma. +notes=FIXME,XXX,TODO diff --git a/setup.py b/setup.py index dddbc61..13af430 100644 --- a/setup.py +++ b/setup.py @@ -5,14 +5,6 @@ import database_files -# try: -# from pypandoc import convert -# read_md = lambda f: convert(f, 'rst') -# except: -# print("Warning: pypandoc module not found, could not convert " -# "Markdown to RST") -# read_md = lambda f: open(f, 'r').read() - CURRENT_DIR = os.path.abspath(os.path.dirname(__file__)) def get_reqs(*fns): @@ -25,85 +17,6 @@ def get_reqs(*fns): lst.append(package.strip()) return lst -# def get_reqs(test=False, pv=None): -# reqs = [ -# 'Django>=1.4', -# 'six>=1.7.2', -# ] -# if test: -# #TODO:remove once Django 1.7 south integration becomes main-stream? -# if not pv: -# reqs.append('South>=1.0') -# elif pv <= 3.2: -# # Note, South dropped Python3 support after 0.8.4... -# reqs.append('South==0.8.4') -# return reqs - -# class TestCommand(Command): -# description = "Runs unittests." -# user_options = [ -# ('name=', None, -# 'Name of the specific test to run.'), -# ('virtual-env-dir=', None, -# 'The location of the virtual environment to use.'), -# ('pv=', None, -# 'The version of Python to use. e.g. 2.7 or 3'), -# ] -# -# def initialize_options(self): -# self.name = None -# self.virtual_env_dir = '.env%s' -# self.pv = 0 -# self.versions = [ -# 2.7, -# 3, -# #3.3,#TODO? -# ] -# -# def finalize_options(self): -# pass -# -# def build_virtualenv(self, pv): -# virtual_env_dir = self.virtual_env_dir % pv -# kwargs = dict(virtual_env_dir=virtual_env_dir, pv=pv) -# if not os.path.isdir(virtual_env_dir): -# cmd = 'virtualenv -p /usr/bin/python{pv} {virtual_env_dir}'.format(**kwargs) -# print(cmd) -# os.system(cmd) -# -# cmd = '{virtual_env_dir}/bin/easy_install -U distribute'.format(**kwargs) -# print(cmd) -# os.system(cmd) -# -# for package in get_reqs(test=True, pv=float(pv)): -# kwargs['package'] = package -# cmd = '{virtual_env_dir}/bin/pip install -U {package}'.format(**kwargs) -# print(cmd) -# os.system(cmd) -# -# def run(self): -# versions = self.versions -# if self.pv: -# versions = [self.pv] -# -# for pv in versions: -# -# self.build_virtualenv(pv) -# kwargs = dict( -# pv=pv, -# virtual_env_dir=self.virtual_env_dir % pv, -# name=self.name) -# -# if self.name: -# cmd = '{virtual_env_dir}/bin/django-admin.py test --pythonpath=. --settings=database_files.tests.settings database_files.tests.tests.{name}'.format(**kwargs) -# else: -# cmd = '{virtual_env_dir}/bin/django-admin.py test --pythonpath=. --settings=database_files.tests.settings database_files.tests'.format(**kwargs) -# -# print(cmd) -# ret = os.system(cmd) -# if ret: -# return - try: long_description = read_md('README.md') except: @@ -117,6 +30,7 @@ def get_reqs(*fns): author='Chris Spencer', author_email='chrisspen@gmail.com', url='http://github.com/chrisspen/django-database-files-3000', + #packages=find_packages(), packages=[ 'database_files', 'database_files.management', @@ -126,16 +40,16 @@ def get_reqs(*fns): ], #https://pypi.python.org/pypi?%3Aaction=list_classifiers classifiers=[ - 'Development Status :: 4 - Beta', + 'Development Status :: 6 - Mature', 'Framework :: Django', 'Intended Audience :: Developers', 'License :: OSI Approved :: BSD License', 'Operating System :: OS Independent', 'Programming Language :: Python', - 'Programming Language :: Python', 'Programming Language :: Python :: 2.7', - 'Programming Language :: Python :: 3.0', - 'Programming Language :: Python :: 3.2', + 'Programming Language :: Python :: 3.3', + 'Programming Language :: Python :: 3.4', + 'Programming Language :: Python :: 3.5', ], install_requires=get_reqs('pip-requirements-min-django.txt', 'pip-requirements.txt'), tests_require=get_reqs('pip-requirements-test.txt'), From de14a38d6ff92ec2cf015ecb56b3fdf1adf2fcc3 Mon Sep 17 00:00:00 2001 From: chrisspen Date: Sat, 15 Oct 2016 22:41:11 -0400 Subject: [PATCH 52/82] Fixed undefined variable. --- database_files/models.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/database_files/models.py b/database_files/models.py index 78ad27e..f34a73b 100644 --- a/database_files/models.py +++ b/database_files/models.py @@ -117,8 +117,8 @@ def dump_files(cls, debug=True, verbose=False): 'system...') % (file_id, name)) f = File.objects.get(id=file_id) write_file( - file.name, - file.content, + f.name, + f.content, overwrite=True) f._content_hash = None f.save() From 2f4cf7b6fd36eb00dcdef9fc50bf23b424389496 Mon Sep 17 00:00:00 2001 From: chrisspen Date: Sat, 15 Oct 2016 22:52:44 -0400 Subject: [PATCH 53/82] Cleaned up setup. --- setup.py | 9 +-------- 1 file changed, 1 insertion(+), 8 deletions(-) diff --git a/setup.py b/setup.py index 13af430..080722c 100644 --- a/setup.py +++ b/setup.py @@ -30,14 +30,7 @@ def get_reqs(*fns): author='Chris Spencer', author_email='chrisspen@gmail.com', url='http://github.com/chrisspen/django-database-files-3000', - #packages=find_packages(), - packages=[ - 'database_files', - 'database_files.management', - 'database_files.management.commands', - 'database_files.migrations', - 'database_files.south_migrations', - ], + packages=find_packages(), #https://pypi.python.org/pypi?%3Aaction=list_classifiers classifiers=[ 'Development Status :: 6 - Mature', From ea49709c4d1381500e4f0575bcdc8b99b2b41deb Mon Sep 17 00:00:00 2001 From: chrisspen Date: Wed, 4 Jan 2017 01:41:23 -0500 Subject: [PATCH 54/82] Updated shields. --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index c75e028..67e765e 100644 --- a/README.md +++ b/README.md @@ -1,7 +1,7 @@ Django Database Files 3000 ========================== -[Build Status](https://travis-ci.org/chrisspen/django-database-files-3000) +[![](https://img.shields.io/pypi/v/django-database-files-3000.svg)](https://pypi.python.org/pypi/django-database-files-3000) [![Build Status](https://img.shields.io/travis/chrisspen/django-database-files-3000.svg?branch=master)](https://travis-ci.org/chrisspen/django-database-files-3000) [![](https://pyup.io/repos/github/chrisspen/django-database-files-3000/shield.svg)](https://pyup.io/repos/github/chrisspen/django-database-files-3000) This is a storage system for Django that stores uploaded files in the database. Files can be served from the database From a4e74ac9f7e3a9043887ef0fecaca9140ce5ad22 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Pawe=C5=82=20Samoraj?= Date: Tue, 7 Feb 2017 01:14:24 +0100 Subject: [PATCH 55/82] Typo fix in README. Typo fix in README. --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 67e765e..c6bfa72 100644 --- a/README.md +++ b/README.md @@ -57,7 +57,7 @@ Settings If false, the file will only exist in the database. -* `DATABASE_FILES_URL_METHOD` = `URL_METHOD_1`|`URL_METHOD_1` (default `URL_METHOD_1`) +* `DATABASE_FILES_URL_METHOD` = `URL_METHOD_1`|`URL_METHOD_2` (default `URL_METHOD_1`) Defines the method to use when rendering the web-accessible URL for a file. From 50584c05ddcf1fd660f5a2665efb1108d76f46ac Mon Sep 17 00:00:00 2001 From: pyup-bot Date: Mon, 6 Feb 2017 19:23:29 -0500 Subject: [PATCH 56/82] Pin south to latest version 1.0.2 --- pip-requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pip-requirements.txt b/pip-requirements.txt index b9700c2..9b072b6 100644 --- a/pip-requirements.txt +++ b/pip-requirements.txt @@ -1,2 +1,2 @@ six>=1.7.2 -South \ No newline at end of file +South==1.0.2 \ No newline at end of file From 7ecee94b749df21fdbf89efc0316773ef6385e6a Mon Sep 17 00:00:00 2001 From: Yamao Maoen Date: Fri, 4 Aug 2017 15:48:20 +0300 Subject: [PATCH 57/82] Drop Django 1.6 support --- .../south_migrations/0001_initial.py | 38 ----------------- .../south_migrations/0002_load_files.py | 30 ------------- ..._datetime__add_field_file__content_hash.py | 42 ------------------- .../south_migrations/0004_set_hash.py | 36 ---------------- database_files/south_migrations/0005_auto.py | 38 ----------------- database_files/south_migrations/__init__.py | 0 pip-requirements-min-django.txt | 2 +- pip-requirements.txt | 3 +- tox.ini | 12 +----- 9 files changed, 3 insertions(+), 198 deletions(-) delete mode 100644 database_files/south_migrations/0001_initial.py delete mode 100644 database_files/south_migrations/0002_load_files.py delete mode 100644 database_files/south_migrations/0003_auto__add_field_file_created_datetime__add_field_file__content_hash.py delete mode 100644 database_files/south_migrations/0004_set_hash.py delete mode 100644 database_files/south_migrations/0005_auto.py delete mode 100644 database_files/south_migrations/__init__.py diff --git a/database_files/south_migrations/0001_initial.py b/database_files/south_migrations/0001_initial.py deleted file mode 100644 index 010f933..0000000 --- a/database_files/south_migrations/0001_initial.py +++ /dev/null @@ -1,38 +0,0 @@ -# encoding: utf-8 -import datetime - -from django.core.management import call_command -from django.db import models - -from south.db import db -from south.v2 import SchemaMigration - -class Migration(SchemaMigration): - - def forwards(self, orm): - - # Adding model 'File' - db.create_table('database_files_file', ( - ('id', self.gf('django.db.models.fields.AutoField')(primary_key=True)), - ('name', self.gf('django.db.models.fields.CharField')(unique=True, max_length=255, db_index=True)), - ('size', self.gf('django.db.models.fields.PositiveIntegerField')()), - ('_content', self.gf('django.db.models.fields.TextField')(db_column='content')), - )) - db.send_create_signal('database_files', ['File']) - - def backwards(self, orm): - - # Deleting model 'File' - db.delete_table('database_files_file') - - models = { - 'database_files.file': { - 'Meta': {'object_name': 'File'}, - '_content': ('django.db.models.fields.TextField', [], {'db_column': "'content'"}), - 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), - 'name': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '255', 'db_index': 'True'}), - 'size': ('django.db.models.fields.PositiveIntegerField', [], {}) - } - } - - complete_apps = ['database_files'] diff --git a/database_files/south_migrations/0002_load_files.py b/database_files/south_migrations/0002_load_files.py deleted file mode 100644 index 58b5a1b..0000000 --- a/database_files/south_migrations/0002_load_files.py +++ /dev/null @@ -1,30 +0,0 @@ -# encoding: utf-8 -import datetime - -from django.core.management import call_command -from django.db import models - -from south.db import db -from south.v2 import DataMigration - -class Migration(DataMigration): - - def forwards(self, orm): - # Load any files referenced by existing models into the database. - call_command('database_files_load') - - def backwards(self, orm): - import database_files - database_files.models.File.objects.all().delete() - - models = { - 'database_files.file': { - 'Meta': {'object_name': 'File'}, - '_content': ('django.db.models.fields.TextField', [], {'db_column': "'content'"}), - 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), - 'name': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '255', 'db_index': 'True'}), - 'size': ('django.db.models.fields.PositiveIntegerField', [], {}) - } - } - - complete_apps = ['database_files'] diff --git a/database_files/south_migrations/0003_auto__add_field_file_created_datetime__add_field_file__content_hash.py b/database_files/south_migrations/0003_auto__add_field_file_created_datetime__add_field_file__content_hash.py deleted file mode 100644 index 1fcf725..0000000 --- a/database_files/south_migrations/0003_auto__add_field_file_created_datetime__add_field_file__content_hash.py +++ /dev/null @@ -1,42 +0,0 @@ -# -*- coding: utf-8 -*- -import datetime -from south.db import db -from south.v2 import SchemaMigration -from django.db import models -from django.utils import timezone - -class Migration(SchemaMigration): - - def forwards(self, orm): - # Adding field 'File.created_datetime' - db.add_column('database_files_file', 'created_datetime', - self.gf('django.db.models.fields.DateTimeField')(default=timezone.now, db_index=True), - keep_default=False) - - # Adding field 'File._content_hash' - db.add_column('database_files_file', '_content_hash', - self.gf('django.db.models.fields.CharField')(max_length=128, null=True, db_column='content_hash', blank=True), - keep_default=False) - - - def backwards(self, orm): - # Deleting field 'File.created_datetime' - db.delete_column('database_files_file', 'created_datetime') - - # Deleting field 'File._content_hash' - db.delete_column('database_files_file', 'content_hash') - - - models = { - 'database_files.file': { - 'Meta': {'object_name': 'File'}, - '_content': ('django.db.models.fields.TextField', [], {'db_column': "'content'"}), - '_content_hash': ('django.db.models.fields.CharField', [], {'max_length': '128', 'null': 'True', 'db_column': "'content_hash'", 'blank': 'True'}), - 'created_datetime': ('django.db.models.fields.DateTimeField', [], {'default': 'timezone.now', 'db_index': 'True'}), - 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), - 'name': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '255', 'db_index': 'True'}), - 'size': ('django.db.models.fields.PositiveIntegerField', [], {}) - } - } - - complete_apps = ['database_files'] \ No newline at end of file diff --git a/database_files/south_migrations/0004_set_hash.py b/database_files/south_migrations/0004_set_hash.py deleted file mode 100644 index 35431b6..0000000 --- a/database_files/south_migrations/0004_set_hash.py +++ /dev/null @@ -1,36 +0,0 @@ -# -*- coding: utf-8 -*- -import datetime -from south.db import db -from south.v2 import DataMigration -from django.db import models -import base64 - -from database_files import utils - -class Migration(DataMigration): - - def forwards(self, orm): - "Write your forwards methods here." - File = orm['database_files.File'] - q = File.objects.all() - for f in q: - f._content_hash = utils.get_text_hash_0004(base64.b64decode(f._content)) - f.save() - - def backwards(self, orm): - "Write your backwards methods here." - - models = { - 'database_files.file': { - 'Meta': {'object_name': 'File'}, - '_content': ('django.db.models.fields.TextField', [], {'db_column': "'content'"}), - '_content_hash': ('django.db.models.fields.CharField', [], {'max_length': '128', 'null': 'True', 'db_column': "'content_hash'", 'blank': 'True'}), - 'created_datetime': ('django.db.models.fields.DateTimeField', [], {'default': 'datetime.datetime.now', 'db_index': 'True'}), - 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), - 'name': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '255', 'db_index': 'True'}), - 'size': ('django.db.models.fields.PositiveIntegerField', [], {}) - } - } - - complete_apps = ['database_files'] - symmetrical = True diff --git a/database_files/south_migrations/0005_auto.py b/database_files/south_migrations/0005_auto.py deleted file mode 100644 index 8501fa5..0000000 --- a/database_files/south_migrations/0005_auto.py +++ /dev/null @@ -1,38 +0,0 @@ -# -*- coding: utf-8 -*- -import datetime -from south.db import db -from south.v2 import SchemaMigration -from django.db import models - - -class Migration(SchemaMigration): - - def forwards(self, orm): - # Adding index on 'File', fields ['_content_hash'] - db.create_index('database_files_file', ['content_hash']) - - # Adding index on 'File', fields ['size'] - db.create_index('database_files_file', ['size']) - - - def backwards(self, orm): - # Removing index on 'File', fields ['size'] - db.delete_index('database_files_file', ['size']) - - # Removing index on 'File', fields ['_content_hash'] - db.delete_index('database_files_file', ['content_hash']) - - - models = { - 'database_files.file': { - 'Meta': {'object_name': 'File'}, - '_content': ('django.db.models.fields.TextField', [], {'db_column': "'content'"}), - '_content_hash': ('django.db.models.fields.CharField', [], {'db_index': 'True', 'max_length': '128', 'null': 'True', 'db_column': "'content_hash'", 'blank': 'True'}), - 'created_datetime': ('django.db.models.fields.DateTimeField', [], {'default': 'datetime.datetime.now', 'db_index': 'True'}), - 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), - 'name': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '255', 'db_index': 'True'}), - 'size': ('django.db.models.fields.PositiveIntegerField', [], {'db_index': 'True'}) - } - } - - complete_apps = ['database_files'] \ No newline at end of file diff --git a/database_files/south_migrations/__init__.py b/database_files/south_migrations/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/pip-requirements-min-django.txt b/pip-requirements-min-django.txt index 2362654..0aff3b6 100644 --- a/pip-requirements-min-django.txt +++ b/pip-requirements-min-django.txt @@ -1,2 +1,2 @@ -Django>=1.4 +Django>=1.7 Django<2 diff --git a/pip-requirements.txt b/pip-requirements.txt index 9b072b6..90e4013 100644 --- a/pip-requirements.txt +++ b/pip-requirements.txt @@ -1,2 +1 @@ -six>=1.7.2 -South==1.0.2 \ No newline at end of file +six>=1.7.2 \ No newline at end of file diff --git a/tox.ini b/tox.ini index 012f1bc..27beaee 100644 --- a/tox.ini +++ b/tox.ini @@ -7,7 +7,7 @@ #https://pypi.python.org/pypi/Django/1.10 # Note, several versions support Python 3.2, but Pip has dropped support, so we can't test them. # See https://github.com/travis-ci/travis-ci/issues/5485 -envlist = py{27,33}-django{15,16},py{27,33,34}-django{17,18},py{27,34,35}-django{19},py{27,34,35}-django{110} +envlist = py{27,33,34}-django{17,18},py{27,34,35}-django{19},py{27,34,35}-django{110} recreate = True [testenv] @@ -20,18 +20,8 @@ basepython = deps = -r{toxinidir}/pip-requirements.txt -r{toxinidir}/pip-requirements-test.txt - django15: Django>=1.5,<1.6 - django16: Django>=1.6,<1.7 django17: Django>=1.7,<1.8 django18: Django>=1.8,<1.9 django19: Django>=1.9,<1.10 django110: Django>=1.10,<2 commands = django-admin.py test --traceback --pythonpath=. --settings=database_files.tests.settings database_files.tests.tests.DatabaseFilesTestCase{env:TESTNAME:} - -# Django 1.5 uses a different test module lookup mechanism, so it needs a different command. -[testenv:py27-django15] -commands = django-admin.py test --traceback --pythonpath=. --settings=database_files.tests.settings tests.DatabaseFilesTestCase{env:TESTNAME:} -#[testenv:py32-django15] -#commands = django-admin.py test --traceback --pythonpath=. --settings=database_files.tests.settings tests.DatabaseFilesTestCase{env:TESTNAME:} -[testenv:py33-django15] -commands = django-admin.py test --traceback --pythonpath=. --settings=database_files.tests.settings tests.DatabaseFilesTestCase{env:TESTNAME:} From a1e5643c1a1486dbd53cade82ee4291ebcf59ceb Mon Sep 17 00:00:00 2001 From: chrisspen Date: Fri, 4 Aug 2017 14:23:51 -0400 Subject: [PATCH 58/82] Extended django version. Updated ignore --- .gitignore | 2 ++ pylint.rc | 4 ++-- tox.ini | 6 ++++-- 3 files changed, 8 insertions(+), 4 deletions(-) diff --git a/.gitignore b/.gitignore index c619fee..dcb81f2 100644 --- a/.gitignore +++ b/.gitignore @@ -57,3 +57,5 @@ docs/_build/ # PIP install version files. /=* +*.geany +*.out diff --git a/pylint.rc b/pylint.rc index dce15c4..0711cd6 100644 --- a/pylint.rc +++ b/pylint.rc @@ -121,7 +121,7 @@ include-ids=yes files-output=no # Tells whether to display a full report or only the messages -reports=yes +reports=no # Python expression which should return a note less than 10 (10 is the highest # note). You have access to the variables errors warning, statement which @@ -336,7 +336,7 @@ ignore-docstrings=yes [FORMAT] # Maximum number of characters on a single line. -max-line-length=100 +max-line-length=160 # Maximum number of lines in a module max-module-lines=1000 diff --git a/tox.ini b/tox.ini index 012f1bc..6a03a20 100644 --- a/tox.ini +++ b/tox.ini @@ -7,7 +7,7 @@ #https://pypi.python.org/pypi/Django/1.10 # Note, several versions support Python 3.2, but Pip has dropped support, so we can't test them. # See https://github.com/travis-ci/travis-ci/issues/5485 -envlist = py{27,33}-django{15,16},py{27,33,34}-django{17,18},py{27,34,35}-django{19},py{27,34,35}-django{110} +envlist = py{27,33}-django{15,16},py{27,33,34}-django{17,18},py{27,34,35}-django{19},py{27,34,35}-django{110},py{27,34,35,36}-django{111} recreate = True [testenv] @@ -17,6 +17,7 @@ basepython = py33: python3.3 py34: python3.4 py35: python3.5 + py36: python3.6 deps = -r{toxinidir}/pip-requirements.txt -r{toxinidir}/pip-requirements-test.txt @@ -25,7 +26,8 @@ deps = django17: Django>=1.7,<1.8 django18: Django>=1.8,<1.9 django19: Django>=1.9,<1.10 - django110: Django>=1.10,<2 + django110: Django>=1.10,<1.11 + django111: Django>=1.11,<2 commands = django-admin.py test --traceback --pythonpath=. --settings=database_files.tests.settings database_files.tests.tests.DatabaseFilesTestCase{env:TESTNAME:} # Django 1.5 uses a different test module lookup mechanism, so it needs a different command. From 39438caa4ddbfbecae1be6961f1a872c60d82d03 Mon Sep 17 00:00:00 2001 From: chrisspen Date: Fri, 4 Aug 2017 14:46:05 -0400 Subject: [PATCH 59/82] Updated travis config to include python 3.6 --- .travis.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.travis.yml b/.travis.yml index 4b861ac..5e3d58a 100644 --- a/.travis.yml +++ b/.travis.yml @@ -2,6 +2,9 @@ language: python python: - "3.5" install: +- sudo add-apt-repository -y ppa:fkrull/deadsnakes +- sudo apt-get -yq update +- sudo apt-get -yq install python3.6 python3.6-dev - pip install tox pylint script: - ./pep8.sh From a1288879ea9a608ada28009c64b3a4296ec5125c Mon Sep 17 00:00:00 2001 From: chrisspen Date: Fri, 4 Aug 2017 14:53:35 -0400 Subject: [PATCH 60/82] Updated travis config to include python 3.6 --- .travis.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.travis.yml b/.travis.yml index 5e3d58a..3022b31 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,3 +1,5 @@ +dist: trusty +sudo: required language: python python: - "3.5" From 05420edb5a819eb78c19ea751a4035eeb75eafde Mon Sep 17 00:00:00 2001 From: chrisspen Date: Fri, 4 Aug 2017 15:07:15 -0400 Subject: [PATCH 61/82] Dropped Python 3.3 support. --- tox.ini | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/tox.ini b/tox.ini index 0abfdf5..46ab0b0 100644 --- a/tox.ini +++ b/tox.ini @@ -1,20 +1,17 @@ [tox] -#https://pypi.python.org/pypi/Django/1.5 -#https://pypi.python.org/pypi/Django/1.6 #https://pypi.python.org/pypi/Django/1.7 #https://pypi.python.org/pypi/Django/1.8 #https://pypi.python.org/pypi/Django/1.9 #https://pypi.python.org/pypi/Django/1.10 # Note, several versions support Python 3.2, but Pip has dropped support, so we can't test them. # See https://github.com/travis-ci/travis-ci/issues/5485 -envlist = py{27,33,34}-django{17,18},py{27,34,35}-django{19},py{27,34,35}-django{110},py{27,34,35,36}-django{111} +envlist = py{27,34}-django{17,18},py{27,34,35}-django{19},py{27,34,35}-django{110},py{27,34,35,36}-django{111} recreate = True [testenv] basepython = py27: python2.7 py32: python3.2 - py33: python3.3 py34: python3.4 py35: python3.5 py36: python3.6 From ba9828f06128238efb645a448ad46dd768bb2bee Mon Sep 17 00:00:00 2001 From: chrisspen Date: Fri, 4 Aug 2017 16:13:17 -0400 Subject: [PATCH 62/82] Updated version --- database_files/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/database_files/__init__.py b/database_files/__init__.py index e3114ec..d4645a1 100644 --- a/database_files/__init__.py +++ b/database_files/__init__.py @@ -1,2 +1,2 @@ -VERSION = (1, 0, 1) +VERSION = (1, 0, 2) __version__ = '.'.join(map(str, VERSION)) From 3f8dd2be467a0ca924fa69ba9133619ad70263b5 Mon Sep 17 00:00:00 2001 From: chrisspen Date: Tue, 31 Oct 2017 22:20:47 -0400 Subject: [PATCH 63/82] Refactored default settings initialization to work with Django 1.11. --- database_files/__init__.py | 2 +- database_files/settings.py | 27 ++++++++------------------- database_files/storage.py | 8 ++++---- database_files/utils.py | 16 ++++++++-------- 4 files changed, 21 insertions(+), 32 deletions(-) diff --git a/database_files/__init__.py b/database_files/__init__.py index d4645a1..81598db 100644 --- a/database_files/__init__.py +++ b/database_files/__init__.py @@ -1,2 +1,2 @@ -VERSION = (1, 0, 2) +VERSION = (1, 0, 3) __version__ = '.'.join(map(str, VERSION)) diff --git a/database_files/settings.py b/database_files/settings.py index 58042ee..aabfb2e 100644 --- a/database_files/settings.py +++ b/database_files/settings.py @@ -5,10 +5,7 @@ # If true, when file objects are created, they will be automatically copied # to the local file system for faster serving. -settings.DB_FILES_AUTO_EXPORT_DB_TO_FS = getattr( - settings, - 'DB_FILES_AUTO_EXPORT_DB_TO_FS', - True) +DB_FILES_AUTO_EXPORT_DB_TO_FS = settings.DB_FILES_AUTO_EXPORT_DB_TO_FS = getattr(settings, 'DB_FILES_AUTO_EXPORT_DB_TO_FS', True) def URL_METHOD_1(name): """ @@ -27,27 +24,19 @@ def URL_METHOD_2(name): ('URL_METHOD_2', URL_METHOD_2), ) -settings.DATABASE_FILES_URL_METHOD_NAME = getattr( - settings, - 'DATABASE_FILES_URL_METHOD', - 'URL_METHOD_1') +DATABASE_FILES_URL_METHOD_NAME = settings.DATABASE_FILES_URL_METHOD_NAME = getattr(settings, 'DATABASE_FILES_URL_METHOD', 'URL_METHOD_1') if callable(settings.DATABASE_FILES_URL_METHOD_NAME): method = settings.DATABASE_FILES_URL_METHOD_NAME else: method = dict(URL_METHODS)[settings.DATABASE_FILES_URL_METHOD_NAME] -settings.DATABASE_FILES_URL_METHOD = method -settings.DB_FILES_DEFAULT_ENFORCE_ENCODING = getattr( - settings, 'DB_FILES_DEFAULT_ENFORCE_ENCODING', True) +DATABASE_FILES_URL_METHOD = settings.DATABASE_FILES_URL_METHOD = method -settings.DB_FILES_DEFAULT_ENCODING = getattr( - settings, - 'DB_FILES_DEFAULT_ENCODING', - 'ascii') +DB_FILES_DEFAULT_ENFORCE_ENCODING = settings.DB_FILES_DEFAULT_ENFORCE_ENCODING = getattr(settings, 'DB_FILES_DEFAULT_ENFORCE_ENCODING', True) -settings.DB_FILES_DEFAULT_ERROR_METHOD = getattr( - settings, 'DB_FILES_DEFAULT_ERROR_METHOD', 'ignore') +DB_FILES_DEFAULT_ENCODING = settings.DB_FILES_DEFAULT_ENCODING = getattr(settings, 'DB_FILES_DEFAULT_ENCODING', 'ascii') -settings.DB_FILES_DEFAULT_HASH_FN_TEMPLATE = getattr( - settings, 'DB_FILES_DEFAULT_HASH_FN_TEMPLATE', '%s.hash') +DB_FILES_DEFAULT_ERROR_METHOD = settings.DB_FILES_DEFAULT_ERROR_METHOD = getattr(settings, 'DB_FILES_DEFAULT_ERROR_METHOD', 'ignore') + +DB_FILES_DEFAULT_HASH_FN_TEMPLATE = settings.DB_FILES_DEFAULT_HASH_FN_TEMPLATE = getattr(settings, 'DB_FILES_DEFAULT_HASH_FN_TEMPLATE', '%s.hash') diff --git a/database_files/storage.py b/database_files/storage.py index e4ea8bc..f7b4ee4 100644 --- a/database_files/storage.py +++ b/database_files/storage.py @@ -11,6 +11,7 @@ from database_files import models from database_files import utils +from database_files import settings as _settings class DatabaseStorage(FileSystemStorage): @@ -32,8 +33,7 @@ def _open(self, name, mode='rb'): f = models.File.objects.get_from_name(name) content = f.content size = f.size - if settings.DB_FILES_AUTO_EXPORT_DB_TO_FS \ - and not utils.is_fresh(f.name, f.content_hash): + if _settings.DB_FILES_AUTO_EXPORT_DB_TO_FS and not utils.is_fresh(f.name, f.content_hash): # Automatically write the file to the filesystem # if it's missing and exists in the database. # This happens if we're using multiple web servers connected @@ -80,7 +80,7 @@ def _save(self, name, content): name=name, ) # Automatically write the change to the local file system. - if settings.DB_FILES_AUTO_EXPORT_DB_TO_FS: + if _settings.DB_FILES_AUTO_EXPORT_DB_TO_FS: utils.write_file(name, content, overwrite=True) #TODO:add callback to handle custom save behavior? return self._generate_name(name, f.pk) @@ -111,7 +111,7 @@ def url(self, name): """ Returns the web-accessible URL for the file with filename `name`. """ - return settings.DATABASE_FILES_URL_METHOD(name) + return _settings.DATABASE_FILES_URL_METHOD(name) def size(self, name): """ diff --git a/database_files/utils.py b/database_files/utils.py index 41ba845..184a476 100644 --- a/database_files/utils.py +++ b/database_files/utils.py @@ -5,6 +5,7 @@ import six from django.conf import settings +from database_files import settings as _settings def is_fresh(name, content_hash): """ @@ -43,8 +44,7 @@ def get_hash_fn(name): if not os.path.isdir(dirs): os.makedirs(dirs) fqfn_parts = os.path.split(fqfn) - hash_fn = os.path.join(fqfn_parts[0], - settings.DB_FILES_DEFAULT_HASH_FN_TEMPLATE % fqfn_parts[1]) + hash_fn = os.path.join(fqfn_parts[0], _settings.DB_FILES_DEFAULT_HASH_FN_TEMPLATE % fqfn_parts[1]) return hash_fn def write_file(name, content, overwrite=False): @@ -88,11 +88,11 @@ def get_file_hash(fin, force_encoding=None, encoding=None, errors=None, chunk_si Iteratively builds a file hash without loading the entire file into memory. """ - force_encoding = force_encoding or settings.DB_FILES_DEFAULT_ENFORCE_ENCODING + force_encoding = force_encoding or _settings.DB_FILES_DEFAULT_ENFORCE_ENCODING - encoding = encoding or settings.DB_FILES_DEFAULT_ENCODING + encoding = encoding or _settings.DB_FILES_DEFAULT_ENCODING - errors = errors or settings.DB_FILES_DEFAULT_ERROR_METHOD + errors = errors or _settings.DB_FILES_DEFAULT_ERROR_METHOD if isinstance(fin, six.string_types): fin = open(fin, 'rb') @@ -124,11 +124,11 @@ def get_text_hash(text, force_encoding=None, encoding=None, errors=None): Returns the hash of the given text. """ - force_encoding = force_encoding or settings.DB_FILES_DEFAULT_ENFORCE_ENCODING + force_encoding = force_encoding or _settings.DB_FILES_DEFAULT_ENFORCE_ENCODING - encoding = encoding or settings.DB_FILES_DEFAULT_ENCODING + encoding = encoding or _settings.DB_FILES_DEFAULT_ENCODING - errors = errors or settings.DB_FILES_DEFAULT_ERROR_METHOD + errors = errors or _settings.DB_FILES_DEFAULT_ERROR_METHOD h = hashlib.sha512() if force_encoding: From 882a90ea4bccbc1de505920747d4d735e340b29d Mon Sep 17 00:00:00 2001 From: Jordan Reiter Date: Thu, 21 Mar 2019 13:44:00 -0400 Subject: [PATCH 64/82] trusty does not have 3.5 distribution Switching to xenial as recommended here: https://travis-ci.community/t/unable-to-download-python-3-7-archive-on-travis-ci/639 --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 3022b31..a2fbdd0 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,4 +1,4 @@ -dist: trusty +dist: xenial sudo: required language: python python: From 0e515f165ef75c6873897e6a4857d4bdd6dd8809 Mon Sep 17 00:00:00 2001 From: Jordan Reiter Date: Thu, 21 Mar 2019 13:51:13 -0400 Subject: [PATCH 65/82] Python 3.4 is EOL & no longer available & pylint > 2.0 does not support 2.7 --- .travis.yml | 2 +- tox.ini | 3 +-- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/.travis.yml b/.travis.yml index a2fbdd0..5834570 100644 --- a/.travis.yml +++ b/.travis.yml @@ -7,7 +7,7 @@ install: - sudo add-apt-repository -y ppa:fkrull/deadsnakes - sudo apt-get -yq update - sudo apt-get -yq install python3.6 python3.6-dev -- pip install tox pylint +- pip install tox "pylint<2.0" script: - ./pep8.sh - tox diff --git a/tox.ini b/tox.ini index 46ab0b0..9a51d60 100644 --- a/tox.ini +++ b/tox.ini @@ -5,14 +5,13 @@ #https://pypi.python.org/pypi/Django/1.10 # Note, several versions support Python 3.2, but Pip has dropped support, so we can't test them. # See https://github.com/travis-ci/travis-ci/issues/5485 -envlist = py{27,34}-django{17,18},py{27,34,35}-django{19},py{27,34,35}-django{110},py{27,34,35,36}-django{111} +envlist = py{27}-django{17,18},py{27,35}-django{19},py{27,35}-django{110},py{27,35,36}-django{111} recreate = True [testenv] basepython = py27: python2.7 py32: python3.2 - py34: python3.4 py35: python3.5 py36: python3.6 deps = From 1a564fd59f235bf07d31c2f539c54a2efbc63253 Mon Sep 17 00:00:00 2001 From: Jordan Reiter Date: Mon, 25 Mar 2019 14:35:12 -0400 Subject: [PATCH 66/82] return None to fix inconsistent-return-statements warning for pylint --- database_files/storage.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/database_files/storage.py b/database_files/storage.py index f7b4ee4..c7429c9 100644 --- a/database_files/storage.py +++ b/database_files/storage.py @@ -53,7 +53,7 @@ def _open(self, name, mode='rb'): size = fh.size else: # Otherwise we don't know where the file is. - return + return None # Normalize the content to a new file object. #fh = StringIO(content) fh = six.BytesIO(content) From e7934c170a5c90b551d1201bea8f277db6a48363 Mon Sep 17 00:00:00 2001 From: Jordan Reiter Date: Thu, 11 Apr 2019 12:06:57 -0400 Subject: [PATCH 67/82] Import location changed in Django 2.0+ --- database_files/settings.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/database_files/settings.py b/database_files/settings.py index aabfb2e..177e0e5 100644 --- a/database_files/settings.py +++ b/database_files/settings.py @@ -1,7 +1,11 @@ import os from django.conf import settings -from django.core.urlresolvers import reverse + +try: + from django.core.urlresolvers import reverse +except ImportError: + from django.urls import reverse # If true, when file objects are created, they will be automatically copied # to the local file system for faster serving. From 034b007ea2a43ea32b00fb688ce37f81941c17d6 Mon Sep 17 00:00:00 2001 From: Jordan Reiter Date: Thu, 11 Apr 2019 12:06:57 -0400 Subject: [PATCH 68/82] Import location changed in Django 2.0+ --- database_files/settings.py | 6 +++++- database_files/storage.py | 5 ++++- 2 files changed, 9 insertions(+), 2 deletions(-) diff --git a/database_files/settings.py b/database_files/settings.py index aabfb2e..177e0e5 100644 --- a/database_files/settings.py +++ b/database_files/settings.py @@ -1,7 +1,11 @@ import os from django.conf import settings -from django.core.urlresolvers import reverse + +try: + from django.core.urlresolvers import reverse +except ImportError: + from django.urls import reverse # If true, when file objects are created, they will be automatically copied # to the local file system for faster serving. diff --git a/database_files/storage.py b/database_files/storage.py index c7429c9..faca37f 100644 --- a/database_files/storage.py +++ b/database_files/storage.py @@ -7,7 +7,10 @@ from django.conf import settings from django.core import files from django.core.files.storage import FileSystemStorage -from django.core.urlresolvers import reverse +try: + from django.core.urlresolvers import reverse +except ImportError: + from django.urls import reverse from database_files import models from database_files import utils From 63d908c759075c9d81db892b44076366cbc3b56d Mon Sep 17 00:00:00 2001 From: Jordan Reiter Date: Wed, 17 Apr 2019 19:55:40 -0400 Subject: [PATCH 69/82] Django requirement --- pip-requirements-min-django.txt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pip-requirements-min-django.txt b/pip-requirements-min-django.txt index 0aff3b6..3fdc35b 100644 --- a/pip-requirements-min-django.txt +++ b/pip-requirements-min-django.txt @@ -1,2 +1,2 @@ -Django>=1.7 -Django<2 +Django>=1.11 +Django<2.3 From 1d54005be721287c7f55347db326fc7ddb6d7c00 Mon Sep 17 00:00:00 2001 From: Jordan Reiter Date: Tue, 30 Apr 2019 16:52:33 -0400 Subject: [PATCH 70/82] convert bytes to str before saving to TextField --- database_files/models.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/database_files/models.py b/database_files/models.py index f34a73b..62cb666 100644 --- a/database_files/models.py +++ b/database_files/models.py @@ -68,7 +68,11 @@ def content(self): @content.setter def content(self, v): - self._content = base64.b64encode(v) + c = base64.b64encode(v) + if not isinstance(c, six.string_types): + c = c.decode('utf-8') + self._content = c + @property def content_hash(self): From 91c2abc07ebd2b8b44b725cd119833e0aa21e9f1 Mon Sep 17 00:00:00 2001 From: Jordan Reiter Date: Tue, 30 Apr 2019 16:56:16 -0400 Subject: [PATCH 71/82] Upgraded tests --- tox.ini | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/tox.ini b/tox.ini index 9a51d60..102a8f3 100644 --- a/tox.ini +++ b/tox.ini @@ -5,21 +5,19 @@ #https://pypi.python.org/pypi/Django/1.10 # Note, several versions support Python 3.2, but Pip has dropped support, so we can't test them. # See https://github.com/travis-ci/travis-ci/issues/5485 -envlist = py{27}-django{17,18},py{27,35}-django{19},py{27,35}-django{110},py{27,35,36}-django{111} +envlist = py{27,35,36}-django{111},py{35,36}-django{20,21,22} recreate = True [testenv] basepython = py27: python2.7 - py32: python3.2 py35: python3.5 py36: python3.6 deps = -r{toxinidir}/pip-requirements.txt -r{toxinidir}/pip-requirements-test.txt - django17: Django>=1.7,<1.8 - django18: Django>=1.8,<1.9 - django19: Django>=1.9,<1.10 - django110: Django>=1.10,<1.11 django111: Django>=1.11,<2 + django20: Django>=2.0<2.1 + django21: Django>=2.1<2.2 + django22: Django>=2.2<2.3 commands = django-admin.py test --traceback --pythonpath=. --settings=database_files.tests.settings database_files.tests.tests.DatabaseFilesTestCase{env:TESTNAME:} From 58f32e85f1c66a1b7993bd94b7bfa46c29e64a09 Mon Sep 17 00:00:00 2001 From: chrisspen Date: Tue, 7 May 2019 15:03:06 -0400 Subject: [PATCH 72/82] Increased version. --- database_files/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/database_files/__init__.py b/database_files/__init__.py index 81598db..65f9591 100644 --- a/database_files/__init__.py +++ b/database_files/__init__.py @@ -1,2 +1,2 @@ -VERSION = (1, 0, 3) +VERSION = (1, 0, 4) __version__ = '.'.join(map(str, VERSION)) From 2b1f98e0bc73a1bf58e71ae457c83ff6a07074be Mon Sep 17 00:00:00 2001 From: chrisspen Date: Thu, 5 Sep 2019 13:10:05 -0400 Subject: [PATCH 73/82] Added yapf config. --- .pre-commit-config.yaml | 6 ++ .style.yapf | 13 ++++ .yapfignore | 1 + database_files/__init__.py | 2 +- .../commands/database_files_cleanup.py | 13 ++-- .../commands/database_files_dump.py | 8 +-- .../commands/database_files_load.py | 17 ++--- .../commands/database_files_rehash.py | 15 +++-- database_files/manager.py | 2 + database_files/models.py | 66 +++++++------------ database_files/settings.py | 3 + database_files/storage.py | 15 +++-- database_files/tests/models.py | 1 + database_files/tests/settings.py | 9 ++- database_files/tests/tests.py | 41 ++++++------ database_files/urls.py | 15 ++--- database_files/utils.py | 35 +++++----- database_files/views.py | 3 +- format-yapf-changed.sh | 11 ++++ format-yapf.sh | 4 ++ pip-requirements-test.txt | 4 ++ setup.py | 2 + 22 files changed, 155 insertions(+), 131 deletions(-) create mode 100644 .pre-commit-config.yaml create mode 100644 .style.yapf create mode 100644 .yapfignore create mode 100755 format-yapf-changed.sh create mode 100755 format-yapf.sh diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml new file mode 100644 index 0000000..6634d8c --- /dev/null +++ b/.pre-commit-config.yaml @@ -0,0 +1,6 @@ +repos: +- repo: https://github.com/pre-commit/mirrors-yapf + rev: v0.28.0 + hooks: + - id: yapf + args: [--in-place, --parallel, --recursive] diff --git a/.style.yapf b/.style.yapf new file mode 100644 index 0000000..e5132c7 --- /dev/null +++ b/.style.yapf @@ -0,0 +1,13 @@ +[style] +BASED_ON_STYLE = pep8 +COLUMN_LIMIT = 160 +COALESCE_BRACKETS = true +DEDENT_CLOSING_BRACKETS = true +BLANK_LINE_BEFORE_NESTED_CLASS_OR_DEF = true +SPACES_BEFORE_COMMENT = 1 +SPLIT_COMPLEX_COMPREHENSION = true +SPACE_BETWEEN_ENDING_COMMA_AND_CLOSING_BRACKET = false +SPLIT_PENALTY_FOR_ADDED_LINE_SPLIT = 10 +CONTINUATION_INDENT_WIDTH = 4 +INDENT_WIDTH = 4 +CONTINUATION_ALIGN_STYLE = SPACE diff --git a/.yapfignore b/.yapfignore new file mode 100644 index 0000000..95512dd --- /dev/null +++ b/.yapfignore @@ -0,0 +1 @@ +database_files/migrations/* diff --git a/database_files/__init__.py b/database_files/__init__.py index 65f9591..99159b8 100644 --- a/database_files/__init__.py +++ b/database_files/__init__.py @@ -1,2 +1,2 @@ -VERSION = (1, 0, 4) +VERSION = (1, 0, 5) __version__ = '.'.join(map(str, VERSION)) diff --git a/database_files/management/commands/database_files_cleanup.py b/database_files/management/commands/database_files_cleanup.py index 178c630..dfbb07b 100644 --- a/database_files/management/commands/database_files_cleanup.py +++ b/database_files/management/commands/database_files_cleanup.py @@ -10,6 +10,7 @@ from database_files.models import File + class Command(BaseCommand): args = '' help = 'Deletes all files in the database that are not referenced by ' + \ @@ -31,11 +32,7 @@ def handle(self, *args, **options): settings.DEBUG = False names = set() dryrun = options['dryrun'] - filenames = set([ - _.strip() - for _ in options['filenames'].split(',') - if _.strip() - ]) + filenames = set([_.strip() for _ in options['filenames'].split(',') if _.strip()]) try: for model in get_models(): print('Checking model %s...' % (model,)) @@ -43,8 +40,8 @@ def handle(self, *args, **options): if not isinstance(field, (FileField, ImageField)): continue # Ignore records with null or empty string values. - q = {'%s__isnull'%field.name:False} - xq = {field.name:''} + q = {'%s__isnull' % field.name: False} + xq = {field.name: ''} subq = model.objects.filter(**q).exclude(**xq) subq_total = subq.count() subq_i = 0 @@ -58,7 +55,7 @@ def handle(self, *args, **options): if not f.name: continue names.add(f.name) - + # Find all database files with names not in our list. print('Finding orphaned files...') orphan_files = File.objects.exclude(name__in=names) diff --git a/database_files/management/commands/database_files_dump.py b/database_files/management/commands/database_files_dump.py index 36534c9..903d056 100644 --- a/database_files/management/commands/database_files_dump.py +++ b/database_files/management/commands/database_files_dump.py @@ -5,11 +5,12 @@ from database_files.models import File + class Command(BaseCommand): option_list = BaseCommand.option_list + ( -# make_option('-w', '--overwrite', action='store_true', -# dest='overwrite', default=False, -# help='If given, overwrites any existing files.'), + # make_option('-w', '--overwrite', action='store_true', + # dest='overwrite', default=False, + # help='If given, overwrites any existing files.'), ) help = 'Dumps all files in the database referenced by FileFields ' + \ 'or ImageFields onto the filesystem in the directory specified by ' + \ @@ -17,4 +18,3 @@ class Command(BaseCommand): def handle(self, *args, **options): File.dump_files(verbose=True) - \ No newline at end of file diff --git a/database_files/management/commands/database_files_load.py b/database_files/management/commands/database_files_load.py index 835e321..c198175 100644 --- a/database_files/management/commands/database_files_load.py +++ b/database_files/management/commands/database_files_load.py @@ -8,11 +8,10 @@ from django.core.management.base import BaseCommand, CommandError from django.db.models import FileField, ImageField, get_models + class Command(BaseCommand): option_list = BaseCommand.option_list + ( - make_option('-m', '--models', - dest='models', default='', - help='A list of models to search for file fields. Default is all.'), + make_option('-m', '--models', dest='models', default='', help='A list of models to search for file fields. Default is all.'), ) help = 'Loads all files on the filesystem referenced by FileFields ' + \ 'or ImageFields into the database. This should only need to be ' + \ @@ -20,11 +19,7 @@ class Command(BaseCommand): def handle(self, *args, **options): show_files = int(options.get('verbosity', 1)) >= 2 - all_models = [ - _.lower().strip() - for _ in options.get('models', '').split() - if _.strip() - ] + all_models = [_.lower().strip() for _ in options.get('models', '').split() if _.strip()] tmp_debug = settings.DEBUG settings.DEBUG = False try: @@ -40,8 +35,8 @@ def handle(self, *args, **options): if show_files: print(model.__name__, field.name) # Ignore records with null or empty string values. - q = {'%s__isnull'%field.name:False} - xq = {field.name:''} + q = {'%s__isnull' % field.name: False} + xq = {field.name: ''} for row in model.objects.filter(**q).exclude(**xq): try: f = getattr(row, field.name) @@ -61,7 +56,7 @@ def handle(self, *args, **options): except IOError: broken += 1 if show_files: - print('-'*80) + print('-' * 80) print('%i broken' % (broken,)) finally: settings.DEBUG = tmp_debug diff --git a/database_files/management/commands/database_files_rehash.py b/database_files/management/commands/database_files_rehash.py index ab23bb0..96d705f 100644 --- a/database_files/management/commands/database_files_rehash.py +++ b/database_files/management/commands/database_files_rehash.py @@ -7,18 +7,19 @@ from database_files.models import File + class Command(BaseCommand): args = ' ... ' help = 'Regenerates hashes for files. If no filenames given, ' + \ 'rehashes everything.' option_list = BaseCommand.option_list + ( -# make_option('--dryrun', -# action='store_true', -# dest='dryrun', -# default=False, -# help='If given, only displays the names of orphaned files ' + \ -# 'and does not delete them.'), - ) + # make_option('--dryrun', + # action='store_true', + # dest='dryrun', + # default=False, + # help='If given, only displays the names of orphaned files ' + \ + # 'and does not delete them.'), + ) def handle(self, *args, **options): tmp_debug = settings.DEBUG diff --git a/database_files/manager.py b/database_files/manager.py index 11d15f9..3949f23 100644 --- a/database_files/manager.py +++ b/database_files/manager.py @@ -2,6 +2,8 @@ from django.db import models + class FileManager(models.Manager): + def get_from_name(self, name): return self.get(name=name) diff --git a/database_files/models.py b/database_files/models.py index 62cb666..86d1da0 100644 --- a/database_files/models.py +++ b/database_files/models.py @@ -14,58 +14,44 @@ from . import settings as _settings + class File(models.Model): - + objects = FileManager() - - name = models.CharField( - max_length=255, - unique=True, - blank=False, - null=False, - db_index=True) - - size = models.PositiveIntegerField( - db_index=True, - blank=False, - null=False) + + name = models.CharField(max_length=255, unique=True, blank=False, null=False, db_index=True) + + size = models.PositiveIntegerField(db_index=True, blank=False, null=False) _content = models.TextField(db_column='content') - - created_datetime = models.DateTimeField( - db_index=True, - default=timezone.now, - verbose_name="Created datetime") - - _content_hash = models.CharField( - db_column='content_hash', - db_index=True, - max_length=128, - blank=True, null=True) - + + created_datetime = models.DateTimeField(db_index=True, default=timezone.now, verbose_name="Created datetime") + + _content_hash = models.CharField(db_column='content_hash', db_index=True, max_length=128, blank=True, null=True) + class Meta: db_table = 'database_files_file' - + def save(self, *args, **kwargs): - + # Check for and clear old content hash. if self.id: old = File.objects.get(id=self.id) if old._content != self._content: self._content_hash = None - + # Recalculate new content hash. self.content_hash - + return super(File, self).save(*args, **kwargs) - + @property def content(self): c = self._content if not isinstance(c, six.binary_type): c = c.encode('utf-8') return base64.b64decode(c) - + @content.setter def content(self, v): c = base64.b64encode(v) @@ -73,13 +59,12 @@ def content(self, v): c = c.decode('utf-8') self._content = c - @property def content_hash(self): if not self._content_hash and self._content: self._content_hash = utils.get_text_hash(self.content) return self._content_hash - + def dump(self, check_hash=False): """ Writes the file content to the filesystem. @@ -88,14 +73,11 @@ def dump(self, check_hash=False): """ if is_fresh(self.name, self._content_hash): return - write_file( - self.name, - self.content, - overwrite=True) + write_file(self.name, self.content, overwrite=True) if check_hash: self._content_hash = None self.save() - + @classmethod def dump_files(cls, debug=True, verbose=False): """ @@ -117,13 +99,9 @@ def dump_files(cls, debug=True, verbose=False): print('%i of %i' % (i, total)) if not is_fresh(name=name, content_hash=content_hash): if verbose: - print(('File %i-%s is stale. Writing to local file ' - 'system...') % (file_id, name)) + print(('File %i-%s is stale. Writing to local file ' 'system...') % (file_id, name)) f = File.objects.get(id=file_id) - write_file( - f.name, - f.content, - overwrite=True) + write_file(f.name, f.content, overwrite=True) f._content_hash = None f.save() finally: diff --git a/database_files/settings.py b/database_files/settings.py index 177e0e5..22600c1 100644 --- a/database_files/settings.py +++ b/database_files/settings.py @@ -11,18 +11,21 @@ # to the local file system for faster serving. DB_FILES_AUTO_EXPORT_DB_TO_FS = settings.DB_FILES_AUTO_EXPORT_DB_TO_FS = getattr(settings, 'DB_FILES_AUTO_EXPORT_DB_TO_FS', True) + def URL_METHOD_1(name): """ Construct file URL based on media URL. """ return os.path.join(settings.MEDIA_URL, name) + def URL_METHOD_2(name): """ Construct file URL based on configured URL pattern. """ return reverse('database_file', kwargs={'name': name}) + URL_METHODS = ( ('URL_METHOD_1', URL_METHOD_1), ('URL_METHOD_2', URL_METHOD_2), diff --git a/database_files/storage.py b/database_files/storage.py index faca37f..6e48a78 100644 --- a/database_files/storage.py +++ b/database_files/storage.py @@ -16,8 +16,9 @@ from database_files import utils from database_files import settings as _settings + class DatabaseStorage(FileSystemStorage): - + def _generate_name(self, name, pk): """ Replaces the filename with the specified pk and removes any dir @@ -26,7 +27,7 @@ def _generate_name(self, name, pk): #file_root, file_ext = os.path.splitext(file_name) #return '%s%s' % (pk, file_name) return name - + def _open(self, name, mode='rb'): """ Open file with filename `name` from the database. @@ -65,7 +66,7 @@ def _open(self, name, mode='rb'): fh.size = size o = files.File(fh) return o - + def _save(self, name, content): """ Save file with filename `name` and given content to the database. @@ -87,7 +88,7 @@ def _save(self, name, content): utils.write_file(name, content, overwrite=True) #TODO:add callback to handle custom save behavior? return self._generate_name(name, f.pk) - + def exists(self, name): """ Returns true if a file with the given filename exists in the database. @@ -96,7 +97,7 @@ def exists(self, name): if models.File.objects.filter(name=name).exists(): return True return super(DatabaseStorage, self).exists(name) - + def delete(self, name): """ Deletes the file with filename `name` from the database and filesystem. @@ -109,13 +110,13 @@ def delete(self, name): except models.File.DoesNotExist: pass return super(DatabaseStorage, self).delete(name) - + def url(self, name): """ Returns the web-accessible URL for the file with filename `name`. """ return _settings.DATABASE_FILES_URL_METHOD(name) - + def size(self, name): """ Returns the size of the file with filename `name` in bytes. diff --git a/database_files/tests/models.py b/database_files/tests/models.py index 428fcfd..5848c09 100644 --- a/database_files/tests/models.py +++ b/database_files/tests/models.py @@ -1,4 +1,5 @@ from django.db import models + class Thing(models.Model): upload = models.FileField(upload_to='i/special') diff --git a/database_files/tests/settings.py b/database_files/tests/settings.py index 1abd6db..b9753d2 100644 --- a/database_files/tests/settings.py +++ b/database_files/tests/settings.py @@ -4,7 +4,7 @@ PROJECT_DIR = os.path.dirname(__file__) DATABASES = { - 'default':{ + 'default': { 'ENGINE': 'django.db.backends.sqlite3', } } @@ -24,6 +24,7 @@ MEDIA_ROOT = os.path.join(PROJECT_DIR, 'media') + # Disable migrations. # http://stackoverflow.com/a/28560805/247542 class DisableMigrations(object): @@ -33,8 +34,10 @@ def __contains__(self, item): def __getitem__(self, item): return "notmigrations" + + SOUTH_TESTS_MIGRATE = False # <= Django 1.8 -# if django.VERSION > (1, 7, 0): # > Django 1.8 +# if django.VERSION > (1, 7, 0): # > Django 1.8 # MIGRATION_MODULES = DisableMigrations() USE_TZ = True @@ -56,5 +59,5 @@ def __getitem__(self, item): #'django.middleware.transaction.TransactionMiddleware', 'django.contrib.auth.middleware.AuthenticationMiddleware', 'django.contrib.messages.middleware.MessageMiddleware', - 'django.middleware.locale.LocaleMiddleware', + 'django.middleware.locale.LocaleMiddleware', ) diff --git a/database_files/tests/tests.py b/database_files/tests/tests.py index 229091a..67ea344 100644 --- a/database_files/tests/tests.py +++ b/database_files/tests/tests.py @@ -17,16 +17,17 @@ DIR = os.path.abspath(os.path.split(__file__)[0]) + class DatabaseFilesTestCase(TestCase): - + def setUp(self): self.media_dir = os.path.join(DIR, 'media/i/special') if os.path.isdir(self.media_dir): shutil.rmtree(self.media_dir) os.makedirs(self.media_dir) - + def test_adding_file(self): - + # Create default thing storing reference to file # in the local media directory. test_fqfn = os.path.join(self.media_dir, 'test.txt') @@ -36,29 +37,29 @@ def test_adding_file(self): o.upload = test_fn o.save() obj_id = o.id - + # Confirm thing was saved. Thing.objects.update() q = Thing.objects.all() self.assertEqual(q.count(), 1) self.assertEqual(q[0].upload.name, test_fn) - + # Confirm the file only exists on the file system # and hasn't been loaded into the database. q = File.objects.all() self.assertEqual(q.count(), 0) - + # Verify we can read the contents of thing. o = Thing.objects.get(id=obj_id) self.assertEqual(o.upload.read(), b"hello there") - + # Verify that by attempting to read the file, we've automatically # loaded it into the database. File.objects.update() q = File.objects.all() self.assertEqual(q.count(), 1) self.assertEqual(q[0].content, b"hello there") - + # Load a dynamically created file outside /media. test_file = files.temp.NamedTemporaryFile( suffix='.txt', @@ -69,9 +70,7 @@ def test_adding_file(self): data0 = b'1234567890' test_file.write(data0) test_file.seek(0) - t = Thing.objects.create( - upload=files.File(test_file), - ) + t = Thing.objects.create(upload=files.File(test_file),) self.assertEqual(File.objects.count(), 2) t = Thing.objects.get(pk=t.pk) self.assertEqual(t.upload.file.size, 10) @@ -79,17 +78,17 @@ def test_adding_file(self): self.assertEqual(t.upload.file.read(), data0) t.upload.delete() self.assertEqual(File.objects.count(), 1) - + # Delete file from local filesystem and re-export it from the database. self.assertEqual(os.path.isfile(test_fqfn), True) os.remove(test_fqfn) self.assertEqual(os.path.isfile(test_fqfn), False) o1.upload.read() # This forces the re-export to the filesystem. self.assertEqual(os.path.isfile(test_fqfn), True) - + # This dumps all files to the filesystem. File.dump_files() - + # Confirm when delete a file from the database, we also delete it from # the filesystem. self.assertEqual(default_storage.exists('i/special/test.txt'), True) @@ -99,12 +98,12 @@ def test_adding_file(self): def test_hash(self): verbose = 1 - + # Create test file. image_content = open(os.path.join(DIR, 'fixtures/test_image.png'), 'rb').read() fqfn = os.path.join(self.media_dir, 'image.png') open(fqfn, 'wb').write(image_content) - + # Calculate hash from various sources and confirm they all match. expected_hash = '35830221efe45ab0dc3d91ca23c29d2d3c20d00c9afeaa096ab256ec322a7a0b3293f07a01377e31060e65b4e5f6f8fdb4c0e56bc586bba5a7ab3e6d6d97a192' # pylint: disable=C0301 h = utils.get_text_hash(image_content) @@ -113,17 +112,17 @@ def test_hash(self): self.assertEqual(h, expected_hash) h = utils.get_text_hash(open(fqfn, 'rb').read()) self.assertEqual(h, expected_hash) -# h = utils.get_text_hash(open(fqfn, 'r').read())#not supported in py3 -# self.assertEqual(h, expected_hash) - + # h = utils.get_text_hash(open(fqfn, 'r').read())#not supported in py3 + # self.assertEqual(h, expected_hash) + # Create test file. if six.PY3: - image_content = six.text_type('aあä')#, encoding='utf-8') + image_content = six.text_type('aあä') #, encoding='utf-8') else: image_content = six.text_type('aあä', encoding='utf-8') fqfn = os.path.join(self.media_dir, 'test.txt') open(fqfn, 'wb').write(image_content.encode('utf-8')) - + expected_hash = '1f40fc92da241694750979ee6cf582f2d5d7d28e18335de05abc54d0560e0f5302860c652bf08d560252aa5e74210546f369fbbbce8c12cfc7957b2652fe9a75' # pylint: disable=C0301 h = utils.get_text_hash(image_content) self.assertEqual(h, expected_hash) diff --git a/database_files/urls.py b/database_files/urls.py index 1721896..c408543 100644 --- a/database_files/urls.py +++ b/database_files/urls.py @@ -1,16 +1,15 @@ - try: # Removed in Django 1.6 from django.conf.urls.defaults import url except ImportError: from django.conf.urls import url - + try: # Relocated in Django 1.6 from django.conf.urls.defaults import pattern except ImportError: # Completely removed in Django 1.10 - try: + try: from django.conf.urls import patterns except ImportError: patterns = None @@ -18,12 +17,10 @@ import database_files.views _patterns = [ -# url(r'^files/(?P.+)$', -# database_files.views.serve, -# name='database_file'), - url(r'^files/(?P.+)$', - database_files.views.serve_mixed, - name='database_file'), + # url(r'^files/(?P.+)$', + # database_files.views.serve, + # name='database_file'), + url(r'^files/(?P.+)$', database_files.views.serve_mixed, name='database_file'), ] if patterns is None: diff --git a/database_files/utils.py b/database_files/utils.py index 184a476..3e363a7 100644 --- a/database_files/utils.py +++ b/database_files/utils.py @@ -1,4 +1,3 @@ - import os import hashlib @@ -7,6 +6,7 @@ from django.conf import settings from database_files import settings as _settings + def is_fresh(name, content_hash): """ Returns true if the file exists on the local filesystem and matches the @@ -14,18 +14,18 @@ def is_fresh(name, content_hash): """ if not content_hash: return False - + # Check that the actual file exists. fqfn = os.path.join(settings.MEDIA_ROOT, name) fqfn = os.path.normpath(fqfn) if not os.path.isfile(fqfn): return False - + # Check for cached hash file. hash_fn = get_hash_fn(name) if os.path.isfile(hash_fn): return open(hash_fn).read().strip() == content_hash - + # Otherwise, calculate the hash of the local file. fqfn = os.path.join(settings.MEDIA_ROOT, name) fqfn = os.path.normpath(fqfn) @@ -34,6 +34,7 @@ def is_fresh(name, content_hash): local_content_hash = get_file_hash(fqfn) return local_content_hash == content_hash + def get_hash_fn(name): """ Returns the filename for the hash file. @@ -47,6 +48,7 @@ def get_hash_fn(name): hash_fn = os.path.join(fqfn_parts[0], _settings.DB_FILES_DEFAULT_HASH_FN_TEMPLATE % fqfn_parts[1]) return hash_fn + def write_file(name, content, overwrite=False): """ Writes the given content to the relative filename under the MEDIA_ROOT. @@ -59,7 +61,7 @@ def write_file(name, content, overwrite=False): if not os.path.isdir(dirs): os.makedirs(dirs) open(fqfn, 'wb').write(content) - + # Cache hash. hash_value = get_file_hash(fqfn) hash_fn = get_hash_fn(name) @@ -69,12 +71,12 @@ def write_file(name, content, overwrite=False): except TypeError: value = hash_value open(hash_fn, 'wb').write(value) - + # Set ownership and permissions. uname = getattr(settings, 'DATABASE_FILES_USER', None) gname = getattr(settings, 'DATABASE_FILES_GROUP', None) if gname: - gname = ':'+gname + gname = ':' + gname if uname: os.system('chown -RL %s%s "%s"' % (uname, gname, dirs)) @@ -83,17 +85,18 @@ def write_file(name, content, overwrite=False): if perms: os.system('chmod -R %s "%s"' % (perms, dirs)) + def get_file_hash(fin, force_encoding=None, encoding=None, errors=None, chunk_size=128): """ Iteratively builds a file hash without loading the entire file into memory. """ - + force_encoding = force_encoding or _settings.DB_FILES_DEFAULT_ENFORCE_ENCODING - + encoding = encoding or _settings.DB_FILES_DEFAULT_ENCODING - + errors = errors or _settings.DB_FILES_DEFAULT_ERROR_METHOD - + if isinstance(fin, six.string_types): fin = open(fin, 'rb') h = hashlib.sha512() @@ -109,6 +112,7 @@ def get_file_hash(fin, force_encoding=None, encoding=None, errors=None, chunk_si h.update(text) return h.hexdigest() + def get_text_hash_0004(text): """ Returns the hash of the given text. @@ -119,17 +123,18 @@ def get_text_hash_0004(text): h.update(text.encode('utf-8', 'replace')) return h.hexdigest() + def get_text_hash(text, force_encoding=None, encoding=None, errors=None): """ Returns the hash of the given text. """ - + force_encoding = force_encoding or _settings.DB_FILES_DEFAULT_ENFORCE_ENCODING - + encoding = encoding or _settings.DB_FILES_DEFAULT_ENCODING - + errors = errors or _settings.DB_FILES_DEFAULT_ERROR_METHOD - + h = hashlib.sha512() if force_encoding: if not isinstance(text, six.text_type): diff --git a/database_files/views.py b/database_files/views.py index 5dd1f05..16d7e91 100644 --- a/database_files/views.py +++ b/database_files/views.py @@ -10,6 +10,7 @@ from database_files.models import File + @cache_control(max_age=86400) def serve(request, name): """ @@ -22,6 +23,7 @@ def serve(request, name): response['Content-Length'] = f.size return response + def serve_mixed(request, *args, **kwargs): """ First attempts to serve the file from the filesystem, @@ -36,4 +38,3 @@ def serve_mixed(request, *args, **kwargs): except Http404: # Then try serving from database. return serve(request, name) - \ No newline at end of file diff --git a/format-yapf-changed.sh b/format-yapf-changed.sh new file mode 100755 index 0000000..35aab0b --- /dev/null +++ b/format-yapf-changed.sh @@ -0,0 +1,11 @@ +#!/bin/bash +FILES=`git status --porcelain | grep -E "*\.py$" | grep -v migration | grep -v "^D " | grep -v "^ D " | grep -v "^R " | awk '{print $2}'` +VENV=${VENV:-.env} +$VENV/bin/yapf --version +if [ -z "$FILES" ] +then + echo "No Python changes detected." +else + echo "Checking: $FILES" + $VENV/bin/yapf --in-place --recursive $FILES +fi diff --git a/format-yapf.sh b/format-yapf.sh new file mode 100755 index 0000000..8cfb390 --- /dev/null +++ b/format-yapf.sh @@ -0,0 +1,4 @@ +#!/bin/bash +# Note, this should be used rarely, and instead the pre-commit hook relied upon. +yapf --in-place --recursive database_files +yapf --in-place --recursive setup.py diff --git a/pip-requirements-test.txt b/pip-requirements-test.txt index cb12429..f55bbf3 100644 --- a/pip-requirements-test.txt +++ b/pip-requirements-test.txt @@ -1 +1,5 @@ tox>=2.0.0 +pylint>=2.2.2 +twine>=1.13.0 +yapf==0.28.0 +pre-commit==1.14.4 diff --git a/setup.py b/setup.py index 080722c..b15b3d3 100644 --- a/setup.py +++ b/setup.py @@ -7,6 +7,7 @@ CURRENT_DIR = os.path.abspath(os.path.dirname(__file__)) + def get_reqs(*fns): lst = [] for fn in fns: @@ -17,6 +18,7 @@ def get_reqs(*fns): lst.append(package.strip()) return lst + try: long_description = read_md('README.md') except: From 4ce86e304a430ebc6b25e19b7ec6eba79356ffe7 Mon Sep 17 00:00:00 2001 From: chrisspen Date: Thu, 5 Sep 2019 13:12:09 -0400 Subject: [PATCH 74/82] Updated readme attachment in setup.py --- setup.py | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/setup.py b/setup.py index b15b3d3..a5b9d68 100644 --- a/setup.py +++ b/setup.py @@ -7,6 +7,13 @@ CURRENT_DIR = os.path.abspath(os.path.dirname(__file__)) +try: + with open(os.path.join(CURRENT_DIR, 'README.md'), encoding='utf-8') as f: + long_description = f.read() +except TypeError: + with open(os.path.join(CURRENT_DIR, 'README.md')) as f: + long_description = f.read() + def get_reqs(*fns): lst = [] @@ -19,16 +26,12 @@ def get_reqs(*fns): return lst -try: - long_description = read_md('README.md') -except: - long_description = '' - setup( name='django-database-files-3000', version=database_files.__version__, description='A storage system for Django that stores uploaded files in both the database and file system.', long_description=long_description, + long_description_content_type='text/markdown', author='Chris Spencer', author_email='chrisspen@gmail.com', url='http://github.com/chrisspen/django-database-files-3000', From bc7b02f59988017c90247192a2c5e603e5f97609 Mon Sep 17 00:00:00 2001 From: chrisspen Date: Thu, 5 Sep 2019 13:53:43 -0400 Subject: [PATCH 75/82] Removed Python2.7 support from test suite. --- setup.py | 3 ++- tox.ini | 4 ++-- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/setup.py b/setup.py index a5b9d68..011ef52 100644 --- a/setup.py +++ b/setup.py @@ -44,10 +44,11 @@ def get_reqs(*fns): 'License :: OSI Approved :: BSD License', 'Operating System :: OS Independent', 'Programming Language :: Python', - 'Programming Language :: Python :: 2.7', 'Programming Language :: Python :: 3.3', 'Programming Language :: Python :: 3.4', 'Programming Language :: Python :: 3.5', + 'Programming Language :: Python :: 3.6', + 'Programming Language :: Python :: 3.7', ], install_requires=get_reqs('pip-requirements-min-django.txt', 'pip-requirements.txt'), tests_require=get_reqs('pip-requirements-test.txt'), diff --git a/tox.ini b/tox.ini index 102a8f3..fa9c680 100644 --- a/tox.ini +++ b/tox.ini @@ -5,14 +5,14 @@ #https://pypi.python.org/pypi/Django/1.10 # Note, several versions support Python 3.2, but Pip has dropped support, so we can't test them. # See https://github.com/travis-ci/travis-ci/issues/5485 -envlist = py{27,35,36}-django{111},py{35,36}-django{20,21,22} +envlist = py{35,36}-django{111},py{35,36,37}-django{20,21,22} recreate = True [testenv] basepython = - py27: python2.7 py35: python3.5 py36: python3.6 + py37: python3.7 deps = -r{toxinidir}/pip-requirements.txt -r{toxinidir}/pip-requirements-test.txt From 0ee1ba9de795106c62ea9139d60a378d10ead26e Mon Sep 17 00:00:00 2001 From: chrisspen Date: Thu, 5 Sep 2019 14:28:28 -0400 Subject: [PATCH 76/82] Removed Python2.7 support from test suite. --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 5834570..9f83e54 100644 --- a/.travis.yml +++ b/.travis.yml @@ -6,7 +6,7 @@ python: install: - sudo add-apt-repository -y ppa:fkrull/deadsnakes - sudo apt-get -yq update -- sudo apt-get -yq install python3.6 python3.6-dev +- sudo apt-get -yq install python3.6 python3.6-dev python3.7 python3.7-dev - pip install tox "pylint<2.0" script: - ./pep8.sh From da50f7dbd60deaa37f033cd2627195573ed87fc8 Mon Sep 17 00:00:00 2001 From: chrisspen Date: Thu, 5 Sep 2019 14:36:43 -0400 Subject: [PATCH 77/82] Added test script. --- .../commands/database_files_cleanup.py | 2 +- database_files/migrations/__init__.py | 20 ------------------- database_files/tests/settings.py | 6 +----- test.sh | 6 ++++++ 4 files changed, 8 insertions(+), 26 deletions(-) create mode 100755 test.sh diff --git a/database_files/management/commands/database_files_cleanup.py b/database_files/management/commands/database_files_cleanup.py index dfbb07b..f6da8ce 100644 --- a/database_files/management/commands/database_files_cleanup.py +++ b/database_files/management/commands/database_files_cleanup.py @@ -32,7 +32,7 @@ def handle(self, *args, **options): settings.DEBUG = False names = set() dryrun = options['dryrun'] - filenames = set([_.strip() for _ in options['filenames'].split(',') if _.strip()]) + filenames = set(_.strip() for _ in options['filenames'].split(',') if _.strip()) try: for model in get_models(): print('Checking model %s...' % (model,)) diff --git a/database_files/migrations/__init__.py b/database_files/migrations/__init__.py index de7bd50..8b13789 100644 --- a/database_files/migrations/__init__.py +++ b/database_files/migrations/__init__.py @@ -1,21 +1 @@ -""" -Django migrations for chroniker app -This package does not contain South migrations. South migrations can be found -in the ``south_migrations`` package. -""" - -SOUTH_ERROR_MESSAGE = """\n -For South support, customize the SOUTH_MIGRATION_MODULES setting like so: - - SOUTH_MIGRATION_MODULES = { - 'database_files': 'database_files.south_migrations', - } -""" - -# Ensure the user is not using Django 1.6 or below with South -try: - from django.db import migrations # noqa -except ImportError: - from django.core.exceptions import ImproperlyConfigured - raise ImproperlyConfigured(SOUTH_ERROR_MESSAGE) diff --git a/database_files/tests/settings.py b/database_files/tests/settings.py index b9753d2..406e893 100644 --- a/database_files/tests/settings.py +++ b/database_files/tests/settings.py @@ -27,7 +27,7 @@ # Disable migrations. # http://stackoverflow.com/a/28560805/247542 -class DisableMigrations(object): +class DisableMigrations: def __contains__(self, item): return True @@ -36,10 +36,6 @@ def __getitem__(self, item): return "notmigrations" -SOUTH_TESTS_MIGRATE = False # <= Django 1.8 -# if django.VERSION > (1, 7, 0): # > Django 1.8 -# MIGRATION_MODULES = DisableMigrations() - USE_TZ = True SECRET_KEY = 'secret' diff --git a/test.sh b/test.sh new file mode 100755 index 0000000..a22d458 --- /dev/null +++ b/test.sh @@ -0,0 +1,6 @@ +#!/bin/bash +# Runs all tests. +set -e +./pep8.sh +rm -Rf .tox +export TESTNAME=; tox From 91d847072f4057aa1e49342267a5411314cb5bca Mon Sep 17 00:00:00 2001 From: chrisspen Date: Thu, 5 Sep 2019 14:43:33 -0400 Subject: [PATCH 78/82] Removed Python2.7 support from test suite. --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 9f83e54..1094bf8 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,4 +1,4 @@ -dist: xenial +dist: bionic sudo: required language: python python: From e7c3406f1f0c32d17058bdde18b42e1e7874f439 Mon Sep 17 00:00:00 2001 From: chrisspen Date: Thu, 5 Sep 2019 14:43:57 -0400 Subject: [PATCH 79/82] Removed Python2.7 support from test suite. --- .travis.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.travis.yml b/.travis.yml index 1094bf8..4fb1d51 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,3 +1,4 @@ +# https://docs.travis-ci.com/user/reference/overview/ dist: bionic sudo: required language: python From 9a44d119cf4782665b97b72fc910cfc559cd3fbb Mon Sep 17 00:00:00 2001 From: chrisspen Date: Thu, 5 Sep 2019 14:48:40 -0400 Subject: [PATCH 80/82] Removed Python2.7 support from test suite. --- .travis.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index 4fb1d51..54a9668 100644 --- a/.travis.yml +++ b/.travis.yml @@ -3,11 +3,11 @@ dist: bionic sudo: required language: python python: -- "3.5" +- "3.6" install: - sudo add-apt-repository -y ppa:fkrull/deadsnakes - sudo apt-get -yq update -- sudo apt-get -yq install python3.6 python3.6-dev python3.7 python3.7-dev +- sudo apt-get -yq install python3.5 python3.5-dev python3.6 python3.6-dev python3.7 python3.7-dev - pip install tox "pylint<2.0" script: - ./pep8.sh From 9b1939f2bd8039b87de92495d0a57845764dd4ef Mon Sep 17 00:00:00 2001 From: chrisspen Date: Thu, 5 Sep 2019 14:55:01 -0400 Subject: [PATCH 81/82] Removed Python2.7 support from test suite. --- .travis.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index 54a9668..792a3d6 100644 --- a/.travis.yml +++ b/.travis.yml @@ -5,10 +5,10 @@ language: python python: - "3.6" install: -- sudo add-apt-repository -y ppa:fkrull/deadsnakes +- sudo add-apt-repository -y ppa:deadsnakes/ppa - sudo apt-get -yq update - sudo apt-get -yq install python3.5 python3.5-dev python3.6 python3.6-dev python3.7 python3.7-dev -- pip install tox "pylint<2.0" +- pip install -r pip-requirements-test.txt script: - ./pep8.sh - tox From 818d5583dd480e31e91a562081a632711f6381f7 Mon Sep 17 00:00:00 2001 From: chrisspen Date: Mon, 28 Oct 2019 13:45:52 -0400 Subject: [PATCH 82/82] Forced name methods to force string. --- database_files/__init__.py | 2 +- database_files/settings.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/database_files/__init__.py b/database_files/__init__.py index 99159b8..2b84fd7 100644 --- a/database_files/__init__.py +++ b/database_files/__init__.py @@ -1,2 +1,2 @@ -VERSION = (1, 0, 5) +VERSION = (1, 0, 6) __version__ = '.'.join(map(str, VERSION)) diff --git a/database_files/settings.py b/database_files/settings.py index 22600c1..db8e27c 100644 --- a/database_files/settings.py +++ b/database_files/settings.py @@ -16,14 +16,14 @@ def URL_METHOD_1(name): """ Construct file URL based on media URL. """ - return os.path.join(settings.MEDIA_URL, name) + return os.path.join(settings.MEDIA_URL, str(name)) def URL_METHOD_2(name): """ Construct file URL based on configured URL pattern. """ - return reverse('database_file', kwargs={'name': name}) + return reverse('database_file', kwargs={'name': str(name)}) URL_METHODS = (