webrofs-fuse

#!/usr/bin/env python3

from __future__ import with_statement

from sys import argv, exit
from threading import Lock
from errno import ENOENT, EREMOTEIO, EIO

import time

from fuse import FUSE, FuseOSError, Operations, LoggingMixIn

import urllib.request
import urllib.error

from os.path import join as pathjoin

CACHE_AGE = 3600


class WebROfs(LoggingMixIn, Operations):
    def __init__(self, urls):
        # download manifest
        self._folders = {}
        self._files = {}
        self._links = {}
        self._cache = {}
        # split urls
        for url in urls.split(","):
            self._load_manifest(url)
        print("Manifest download complete")

    def _load_manifest(self, url):
        print("Download manifest " + url)
        try:
            mfile = url + "/manifest"
            response = urllib.request.urlopen(mfile)
            charset = response.info().get_param('charset', 'utf-8')
            data = response.read().decode(charset)
            for line in data.split("\n"):
                if not line == "":
                    line = line.split("|")
                    if line[0] == "M":
                        # we link another manifest
                        self._load_manifest(line[1])
                    elif line[0] == "F":
                        # if this folder exists, dont empty it!
                        if line[1] not in self._folders:
                            self._folders[line[1]] = []
                            self._files[line[1]] = line[2:] + [url]
                    elif line[0] == "S":
                        # symlink
                        self._links[line[1]] = line[2]
                    else:
                        # this is a file
                        fullpath = pathjoin(line[0], line[1])
                        self._files[fullpath] = line[2:] + [url]
                        # if a file exists in a higher stack, don't make two references in
                        # the directory list
                        if line[1] not in self._folders[line[0]]:
                            self._folders[line[0]].append(line[1])
        except Exception as e:
            print("Could not load manifest for volume " + mfile)
            print(e)
            raise SystemExit()

    def _add_cached(self, path, data, time):
        self._cache[path] = (time, data)

    def _remove_cached(self, path):
        try:
            del self._cache[path]
        except:
            pass

    def _is_cached(self, path):
        if path in self._cache:
            # check it has not passed cache age
            if self._cache[path][0] < (int(time.time()) - CACHE_AGE):
                # possible race condition
                self._remove_cached(path)
                return False
            else:
                return True
        else:
            return False

    def read(self, path, size, offset, fh):
        try:
            if path in self._files:
                if self._is_cached(path):
                    # return cached version
                    try:
                        return self._cache[path][1][offset:(offset + size)]
                    except:
                        # there was a problem getting file from cache, get remote copy
                        return urllib.request.urlopen(self.url + path).read()[offset:(offset + size)]
                else:
                    # data is not cached or old, get remote copy
                    try:
                        # get url for this file
                        try:
                            url = self._files[path][8]
                        except KeyError:
                            raise FuseOSError(ENOENT)
                        # get remote copy
                        data = urllib.request.urlopen(url + path).read()
                    except urllib.error.HTTPError:
                        raise FuseOSError(EREMOTEIO)
                    except Exception as e:
                        # TODO log the uncaught exception
                        raise FuseOSError(EIO)
                    # add to cache and return
                    self._add_cached(path, data, int(time.time()))
                    return data[offset:(offset + size)]
        except Exception as e:
            # TODO log the uncaught exception
            raise FuseOSError(EIO)

    def readlink(self, path):
        try:
            return self._links[path]
        except KeyError:
            raise FuseOSError(ENOENT)
        except Exception as e:
            # TODO log the uncaught exception
            raise FuseOSError(EIO)

    def getattr(self, path, fh=None):
        try:
            return {'st_atime': float(self._files[path][0]), 
                'st_ctime': float(self._files[path][1]),
                'st_gid': int(self._files[path][2]), 
                'st_mode': int(self._files[path][3]),
                'st_mtime': float(self._files[path][4]), 
                'st_nlink': int(self._files[path][5]), 
                'st_size': int(self._files[path][6]), 
                'st_uid': int(self._files[path][7])}
        except KeyError:
            raise FuseOSError(ENOENT)
        except Exception as e:
            # TODO log the uncaught exception
            raise FuseOSError(EIO)

    def readdir(self, path, fh):
        rlist = ['.', '..']
        try:
            for f in self._folders[path]:
                rlist.append(f)
            return rlist
        except KeyError:
            raise FuseOSError(ENOENT)
        except Exception as e:
            # TODO log the uncaught exception
            raise FuseOSError(EIO)


if __name__ == '__main__':
    if len(argv) != 3:
        print('usage: %s <urls seperate by ,> <mountpoint>' % argv[0])
        exit(1)
    fuse = FUSE(WebROfs(argv[1]), argv[2], foreground=True)