sync with develop

.circleci/config.yml

@@ -6,7 +6,7 @@ jobs:
     steps:
       - checkout
       - run: mkdir -p /tmp/coverage && mkdir -p /tmp/test_results
-      - run: pip3 install -e .[dev]
+      - run: pip3 install .
       - run: 
           name: Test
           command: pytest -vv tests

.gitignore

@@ -8,6 +8,8 @@ dev_notes
 !tests/deep_mlp/end_to_end.ipynb
 *.egg*
 worktree/
+build/
+dist/
 # tests
 .pytest_cache
 tests/*/cpp

MANIFEST.in

@@ -1,2 +1,2 @@
 include LICENSE README.md
-recursive-include utensor_cgen/backend/snippets/templates *
+recursive-include utensor_cgen/backend/utensor/snippets/templates *

Makefile

@@ -3,7 +3,7 @@
 tests:
 	rm -f tests_log.txt
 	make test_utils test_ir test_transformer test_frontend \
-		test_matcher test_graph_constructor
+		test_matcher test_graph_constructor test_backend
 
 test_%:
 	@if [ -d .venv ]; then \
@@ -12,5 +12,23 @@ test_%:
 	 	 pytest tests/$@ -vv -s | tee -a tests_log.txt; \
 	 fi;
 
+package:
+	rm -rf dist/*
+	.venv/bin/python setup.py bdist_wheel sdist
+	rm -rf build utensor_cgen.egg-info/
+
+upload-test: package
+	.venv/bin/twine upload -r pypitest dist/*
+
+install-test: package
+	pip uninstall -y utensor-cgen
+	pip install dist/*.tar.gz
+
+upload: package
+	.venv/bin/twine upload -r pypi dist/*
+
 clean:
-	rm -rf tests_log.txt *.pdf .pytest_cache
+	rm -rf tests_log.txt *.pdf \
+	models data \
+	tests/test_backend/{models,data} \
+	.pytest_cache dist/ build/

README.rst

@@ -72,26 +72,22 @@ Overall Architecture
 
 ::
 
-      ============       +-----------------+       ===================
-    || model file || --> | frontend Parser | --> || uTensorGraph (IR) ||
-      ============       +-----------------+       ===================
-                                                               |
-                     +-------------------------------+         |
-                     |       graph transformer       |         |
-                     | (legalization & optimization) | <------/
-                     +-------------------------------+
-                                    |
-                                    v
-                         ===========================
-                       ||       uTensorGraph        ||
-                       || (legalized and optimized) ||
-                         ===========================
-                                       |
-    +--------------------------+       |
-    | backend (code generator) | <----/
+      ============       +-----------------+       ============================
+    || model file || --> | frontend Parser | --> || uTensorGraph (IR, generic) ||
+      ============       +-----------------+       ============================
+                                                                |
+                                                                v
+                                                     +---------------------+
+                        =======================      |  graph transformer  |
+                      ||     uTensorGraph     || <-- |    (optimization)   |
+                      || (generic, optimized) ||     +---------------------+
+                        =======================                                    
+                                     |
+    +--------------------------+     |
+    | backend (code generator) | <--/
     +--------------------------+
          |
-         `---> (target files, ex: model.cpp, model.hpp, weights.idx)
+         `---> (target files, ex: model.cpp, model.hpp, weights.idx, ...etc)
 
 Basic Usage
 ===========
@@ -114,7 +110,8 @@ Convert Model File to C/C++ Code
 .. code-block:: console
 
   $ utensor-cli convert <model.pb> \
-    --output-nodes=<node_name>[,<node_name>,...]
+    --output-nodes=<node name>[,<node name>,...] \
+    [--config=config.toml]
 
 Convert given pb file into cpp/hpp files.
 
@@ -123,6 +120,8 @@ nodes you want to output, seperated by comma for multiple values.
 
 In graph theory terminology, they are ``leaf`` nodes of your graph.
 
+Use ``--config`` to pass a configuration file to the cli, you can use ``generate-config`` command to generate one (see below).
+
 example
 ~~~~~~~
 
@@ -132,8 +131,34 @@ example
 
 Run ``utensor-cli convert --help`` for detailed information.
 
-:mod:`utensor_cgen` as Library
-==============================
+Configuration
+-------------
+
+``utensor-cli`` use ``toml`` as configuration format.
+
+You can generate configuration file of given target as following:
+
+.. code-block:: console
+
+  $ utensor-cli generate-config --target <target name> [-o filename.toml]
+
+This command will generate a ``toml`` file listing all configurable values with its defaults.
+
+You can modify the value and pass the file to cli with ``--config`` flag.
+
+example
+~~~~~~~
+
+.. code-block:: console
+
+  # generate config file
+  $ utensor-cli generate-config --target utensor -o myconfig.toml
+
+  # after editting myconfig.toml
+  $ utensor-cli convert mymodel.pb --config=myconfig.toml --output-nodes=output,...
+
+Use :mod:`utensor_cgen` as Library
+==================================
 
 .. subgraph-match-begine
 
@@ -223,8 +248,8 @@ TensorFlow_
 
 1. Freeze your `tensorflow.Graph`
 
-  - please refer to the `official doc <https://www.tensorflow.org/guide/extend/model_files>`_
-    and read the `Freezing <https://www.tensorflow.org/guide/extend/model_files#freezing>`_ section
+  - please refer to this `issue track <https://github.com/tensorflow/tensorflow/issues/27614>`_ for detail
+  - especially this `comment <https://github.com/tensorflow/tensorflow/issues/27614#issuecomment-571889676>`_ by Robin2091
 
 2. Follow instructions in :ref:`install` section to install :mod:`utensor_cgen`
 

example_config.toml

@@ -0,0 +1,15 @@
+# https://github.com/toml-lang/toml
+# <target_name>.<component>.<part>
+[utensor.backend]
+legacy-api = true
+
+[utensor.backend.legacy_code_generator]
+src_fname = "None"
+params_dir = "data"
+embed_params_dir = "/fs/data"
+model_dir = "models"
+transform_methods = [ "dropout(name_pattern=r\"(dropout[_\\w\\d]*)/.*\")", "linear_reorder", "quantize", "conv_pool", "inline", "biasAdd", "remove_id_op", "fake_gather_v2", "refcnt",]
+save_graph = false
+debug_cmt = false
+
+[utensor.backend.graph_lower]

plugins/dummy_backend/__init__.py

@@ -0,0 +1,34 @@
+from textwrap import wrap
+
+from utensor_cgen.backend.base import Backend
+from utensor_cgen.backend import BackendManager
+from utensor_cgen.utils import class_property
+
+@BackendManager.register
+class DummyBackend(Backend):
+    TARGET = 'dummy-backend'
+
+    def __init__(self, config):
+        if not config:
+            config = self.default_config
+        self.output_file = config[self.TARGET][self.COMPONENT]['output-file']
+
+    def apply(self, ugraph):
+        with open(self.output_file, 'w') as fid:
+            fid.write('#include <stdio.h>\n\n')
+            fid.write('int main(int argc, char* argv[]) {\n')
+            fid.write('    printf("graph name: {}\\n");\n'.format(ugraph.name))
+            fid.write('    printf("ops in topological sorted order:\\n");\n')
+            for op_name in ugraph.topo_order:
+                fid.write('    printf("    {}\\n");\n'.format(op_name))
+            fid.write('    return 0;\n}')
+
+    @class_property
+    def default_config(cls):
+        return {
+            cls.TARGET: {
+                    cls.COMPONENT: {
+                    'output-file': 'list_op.c'
+                }
+            }
+        }

setup.py

@@ -3,27 +3,48 @@
 import os
 
 from setuptools import find_packages, setup
+from setuptools.command.develop import develop as _develop
+from setuptools.command.install import install as _install
 
 root_dir = os.path.abspath(os.path.dirname(__file__))
 with open(os.path.join(root_dir, "LICENSE")) as rf:
     license = rf.read()
 
+
+class _CompileFlatbuffMixin(object):
+
+    def run(self):
+        super(_CompileFlatbuffMixin, self).run()
+        self._build_flatbuffer()
+
+    def _build_flatbuffer(self):
+        install_dir = self.install_platlib
+        if install_dir is None:
+            install_dir = os.path.abspath('utensor')
+
+
+class _Install(_CompileFlatbuffMixin, _install): pass
+
+
+class _Develop(_CompileFlatbuffMixin, _develop): pass
+
+
 setup(
     name='utensor_cgen',
     version_config={
         "starting_version": "0.0.0",
-        "version_format": "{tag}.dev{sha:.7s}"
+        "version_format": "{tag}.{sha:.7s}.dev"
     },
     setup_requires=['better-setuptools-git-version'],
+    cmdclass={'install': _Install, 'develop': _Develop},
     description="C code generation program for uTensor",
     long_description="please go to [doc](https://utensor-cgen.readthedocs.io/en/latest/) page for more information",
-    url="https://github.com/dboyliao/utensor_cgen",
+    url="https://github.com/uTensor/utensor_cgen",
     author="Dboy Liao",
     author_email="[email protected]",
     license=license,
     packages=find_packages(),
-    include_package_data=True,
-    package_data={"utensor_cgen": ["backend/snippets/templates/*"]},
+    package_data={'utensor_cgen.backend.utensor.snippets': ["templates/*/*/*"]},
     entry_points={
         "console_scripts": [
             "utensor-cli=utensor_cgen.cli:cli"
@@ -38,7 +59,8 @@
         'torchvision',
         'onnx-tf==1.2.1',
         'graphviz',
-        'flatbuffers'
+        'flatbuffers',
+        'toml',
     ],
     extras_require={
         'dev': ['pytest']

tests/test_backend/conftest.py

@@ -0,0 +1,14 @@
+import os
+from pytest import fixture
+
+@fixture(scope='session', name='mlp_ugraph')
+def mlp_ugraph():
+    from utensor_cgen.frontend import FrontendSelector
+    model_file = os.path.join(
+        os.path.abspath(
+            os.path.join(
+                os.path.dirname(__file__), '..')
+            ),
+            'deep_mlp/simple_mnist.pb'
+    )
+    return FrontendSelector.parse(model_file, output_nodes=['y_pred'])

tests/test_backend/test_utensor.py

@@ -0,0 +1,19 @@
+import os
+
+def test_legacy_utensor(mlp_ugraph):
+    from utensor_cgen.backend.utensor import uTensorBackend
+
+    this_dir = os.path.dirname(__file__)
+
+    uTensorBackend(config={
+        'utensor': {
+            'backend': {
+                'legacy-api': True,
+                'code_generator': {
+                    'model_dir': os.path.join(this_dir, 'models'),
+                    'params_dir': os.path.join(this_dir, 'data'),
+                },
+                'graph_lower': {}
+            },
+        }
+    }).apply(mlp_ugraph)

tests/test_ir/test_uTensorGraph/test_graph.py

@@ -36,7 +36,7 @@ def test_op_info():
                             output_tensors=[],
                             n_outputs=0,
                             op_type='no_op',
-                            backend='tensorflow',
+                            lib_name='tensorflow',
                             op_attr={
                                 '_utensor_to_skip': [1, 2, 3],
                                 '_utensor_skip_this_too': None,

tests/test_transformer/test_convpool/test_vgg.py

@@ -4,9 +4,9 @@
 def factory():
     def test(vgg_ugraph):
         trans = TransformerPipeline([
-            ('linear_reorder', {}),
-            ('quantize', {}),
-            ('conv_pool', {})
+            'linear_reorder',
+            'quantize',
+            'conv_pool',
         ])
         new_ugraph = trans.transform(vgg_ugraph)
         num_conv = len(vgg_ugraph.get_ops_by_type('Conv2D'))

tests/test_transformer/test_linear_reorder/conftest.py

@@ -11,9 +11,9 @@ def subject_ugraph_1():
         input_1 = tf.placeholder(dtype=tf.float32, shape=[None, 512, 512, 10], name='input_1')
         relu_1 = tf.nn.relu(input_1, name='relu_1')
         max_pool_1 = tf.nn.max_pool(relu_1, ksize=[1, 3, 3, 1], strides=[1, 2, 2, 1], padding='VALID', name='pool_1')
-        input_2 = tf.placeholder(dtype=tf.float32, shape=[None, 512, 512, 3], name='input_2')
-        relu_2 = tf.nn.relu(input_1, name='relu_2')
-        max_pool_2 = tf.nn.max_pool(relu_1, ksize=[1, 3, 3, 1], strides=[1, 2, 2, 1], padding='VALID', name='pool_2')
+        input_2 = tf.placeholder(dtype=tf.float32, shape=[None, 512, 512, 10], name='input_2')
+        relu_2 = tf.nn.relu(input_2, name='relu_2')
+        max_pool_2 = tf.nn.max_pool(relu_2, ksize=[1, 3, 3, 1], strides=[1, 2, 2, 1], padding='VALID', name='pool_2')
         output = tf.add(max_pool_1, max_pool_2, name='output')
-    subj_ugraph = GraphDefParser.parse(graph.as_graph_def(), output_nodes=['output'])
+    subj_ugraph = GraphDefParser.parse(graph.as_graph_def(), output_nodes=[output.op.name])
     return subj_ugraph

tests/test_transformer/test_pipeline/conftest.py

@@ -8,9 +8,11 @@
 
 @pytest.fixture(scope='function', name='methods')
 def pipeline_methods():
-    all_methods = [(BatchNormTransformer.METHOD_NAME, {}),
-                   (DropoutTransformer.METHOD_NAME, {}),
-                   (QuantizeTransformer.METHOD_NAME, {}),
-                   (RefCntOptimizer.METHOD_NAME, {})]
+    all_methods = [
+        BatchNormTransformer.METHOD_NAME,
+        DropoutTransformer.METHOD_NAME,
+        QuantizeTransformer.METHOD_NAME,
+        RefCntOptimizer.METHOD_NAME,
+    ]
     shuffle(all_methods)
     return all_methods

tests/test_transformer/test_pipeline/test_pipeline_transformer.py

@@ -5,17 +5,17 @@
 def test_pipeline_1(methods):
     pipeline = TransformerPipeline(methods)
     assert len(pipeline.pipeline) == len(methods)
-    for transformer, (method_name, _) in zip(pipeline.pipeline, methods):
+    for transformer, method_name in zip(pipeline.pipeline, methods):
         assert isinstance(transformer, pipeline.TRANSFORMER_MAP[method_name])
 
 def test_pipeline_2(methods):
     pipeline = TransformerPipeline(methods)
     assert len(pipeline.pipeline) == len(methods)
-    for transformer, (method_name, _) in zip(pipeline.pipeline, methods):
+    for transformer, method_name in zip(pipeline.pipeline, methods):
         assert isinstance(transformer, pipeline.TRANSFORMER_MAP[method_name])
 
 def test_pipeline_3(methods):
     pipeline = TransformerPipeline(methods)
     assert len(pipeline.pipeline) == len(methods)
-    for transformer, (method_name, _) in zip(pipeline.pipeline, methods):
+    for transformer, method_name in zip(pipeline.pipeline, methods):
         assert isinstance(transformer, pipeline.TRANSFORMER_MAP[method_name])

utensor_cgen/__init__.py

@@ -0,0 +1,12 @@
+import sys
+
+import pkg_resources
+
+from utensor_cgen._extensions import _ExtensionsLoader
+
+__version__ = (
+  pkg_resources
+  .get_distribution('utensor_cgen')
+  .version
+)
+sys.modules['utensor_cgen.extensions'] = _ExtensionsLoader()