Fix submodule error

composability_bench/.gitignore

@@ -0,0 +1,89 @@
+# Byte-compiled / optimized / DLL files
+__pycache__/
+*.py[cod]
+*$py.class
+
+# C extensions
+*.so
+
+# Distribution / packaging
+.Python
+env/
+build/
+develop-eggs/
+dist/
+downloads/
+eggs/
+.eggs/
+lib/
+lib64/
+parts/
+sdist/
+var/
+*.egg-info/
+.installed.cfg
+*.egg
+
+# PyInstaller
+#  Usually these files are written by a python script from a template
+#  before PyInstaller builds the exe, so as to inject date/other infos into it.
+*.manifest
+*.spec
+
+# Installer logs
+pip-log.txt
+pip-delete-this-directory.txt
+
+# Unit test / coverage reports
+htmlcov/
+.tox/
+.coverage
+.coverage.*
+.cache
+nosetests.xml
+coverage.xml
+*,cover
+.hypothesis/
+
+# Translations
+*.mo
+*.pot
+
+# Django stuff:
+*.log
+local_settings.py
+
+# Flask stuff:
+instance/
+.webassets-cache
+
+# Scrapy stuff:
+.scrapy
+
+# Sphinx documentation
+docs/_build/
+
+# PyBuilder
+target/
+
+# IPython Notebook
+.ipynb_checkpoints
+
+# pyenv
+.python-version
+
+# celery beat schedule file
+celerybeat-schedule
+
+# dotenv
+.env
+
+# virtualenv
+venv/
+ENV/
+
+# Spyder project settings
+.spyderproject
+
+# Rope project settings
+.ropeproject

composability_bench/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2017 Intel Corporation
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

composability_bench/README.md

@@ -0,0 +1,23 @@
+# Composability Benchmarks
+Show effects of over-subscription and ways to fix that
+
+## Installation
+Install TBB and SMP modules for Python in order to evaluate benefits of composable multithreading
+```
+conda install -c intel tbb4py smp
+```
+Alternatively, run `set_python_envs.sh` to set up environment with components of Intel Distribution for Python
+
+## Running
+Effects are visible on big enough machine with 32 and more cores.
+Run following modes:
+
+```
+python -m tbb collab_filt.py
+python -m tbb dask_bench2.py
+python -m smp collab_filt.py
+python -m smp dask_bench2.py
+```
+
+## See also
+https://software.intel.com/en-us/blogs/2016/04/04/unleash-parallel-performance-of-python-programs

composability_bench/collab_filt.py

@@ -0,0 +1,140 @@
+#!/usr/bin/env python
+# Copyright (c) 2017, Intel Corporation
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are met:
+#
+#     * Redistributions of source code must retain the above copyright notice,
+#       this list of conditions and the following disclaimer.
+#     * Redistributions in binary form must reproduce the above copyright
+#       notice, this list of conditions and the following disclaimer in the
+#       documentation and/or other materials provided with the distribution.
+#     * Neither the name of Intel Corporation nor the names of its contributors
+#       may be used to endorse or promote products derived from this software
+#       without specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE
+# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+
+import sys
+import time
+import timeit
+import numpy as np
+import dask.array as da
+from dask.diagnostics import ProgressBar
+import random
+import argparse
+import numba
+
+number_of_users = 400000
+features = 3260
+chunk = 10000
+
+try:
+    import numpy.random_intel as rnd
+    numpy_ver="intel"
+except:
+    import numpy.random as rnd
+    numpy_ver="std"
+
+parser = argparse.ArgumentParser(sys.argv[0])
+parser.add_argument('--math', required=False, default='dask_numba', help="Select computation approach: numpy, dask, numpy_numba, dask_numba")
+parser.add_argument('--features', required=False, default=features, help="Number of features to process")
+parser.add_argument('--users', required=False, default=number_of_users, help="Number of users to process")
+parser.add_argument('--chunk', required=False, default=chunk, help="Chunk size for splitting dask arrays")
+parser.add_argument('--verbose', '-v', required=False, default=False, help="show progress information")
+parser.add_argument('--prefix', required=False, default="@", help="Prepend result output with this string")
+args = parser.parse_args()
+
+features = int(args.features)
+number_of_users = int(args.users)
+chunk = int(args.chunk)
+
+print("Generating fake similarity")
+#topk = da.random.normal(size=(features, features), chunks=(features, features)).compute()
+topk = rnd.normal(size=(features, features))
+t = da.from_array(topk, chunks=(features, features))
+
+print("Generating fake user data")
+#users = da.random.normal(size=(features, number_of_users), chunks=(features, chunk)).compute()
+#users = rnd.normal(size=(features, number_of_users))
+users = np.zeros(shape=(features, number_of_users), dtype=np.float64)
+objects_idx = np.arange(features)
+rated = rnd.randint(0, 10, size=number_of_users, dtype=np.int32)
+for user in range(number_of_users):
+    rnd.shuffle(objects_idx)
+    items_rated = rated[user]
+    users[objects_idx[:items_rated], user] = rnd.randint(1, 5, size=items_rated, dtype=np.int32)
+
+u = da.from_array(users, chunks=(features, chunk), name=False)
+
+def run_numpy():
+    x = topk.dot(users)
+    x = np.where(users>0, 0, x)
+    return x.argmax(axis=0)
+
+
+def run_dask():
+    x = t.dot(u)
+    x = da.where(u>0, 0, x)
+    r = x.argmax(axis=0)
+    return r.compute()
+
+
+@numba.guvectorize('(f8[:],f8[:],i4[:])', '(n),(n)->()', nopython=True, target="parallel")
+def recommendation(x, u, r):
+    maxx = x[0]
+    r[0] = -1
+    for i in range(x.shape[0]):
+        if u[i] == 0 and maxx < x[i]: # if user has no rank for the item
+           maxx = x[i]
+           r[0] = i
+
+
+def run_numpy_numba():
+    x = topk.dot(users)
+    return recommendation(x, users)
+
+
+def run_dask_numba():
+    x = t.dot(u)
+    r = da.map_blocks(recommendation, x, u, drop_axis=0)
+    return r.compute()
+
+
+# ======================
+
+# ======================
+
+runner_name= 'run_'+args.math
+if runner_name not in globals():
+    print('--math=', args.math, " is not implemented, running numpy")
+    runner = run_numpy
+else:
+    runner = globals()[runner_name]
+
+if args.verbose:
+    ProgressBar().register()
+
+print("Running recommendation system")
+for i in range(3):
+    tic = time.time()
+    r = runner()
+    toc = time.time()
+    time_diff = toc - tic
+    if args.verbose:
+        print("Result shape: ", r.shape, " strides: ", r.strides, " ", r)
+
+    print("%s run=%d numpy=%s users=%d math=%s, chunk=%d in %.2f sec, %f users/sec" % \
+         (str(args.prefix), i, numpy_ver, number_of_users, args.math, chunk, time_diff,
+          float(number_of_users)/time_diff))
+    sys.stdout.flush()

composability_bench/dask_bench2.py

@@ -0,0 +1,64 @@
+#!/usr/bin/env python
+# Copyright (c) 2017, Intel Corporation
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are met:
+#
+#     * Redistributions of source code must retain the above copyright notice,
+#       this list of conditions and the following disclaimer.
+#     * Redistributions in binary form must reproduce the above copyright
+#       notice, this list of conditions and the following disclaimer in the
+#       documentation and/or other materials provided with the distribution.
+#     * Neither the name of Intel Corporation nor the names of its contributors
+#       may be used to endorse or promote products derived from this software
+#       without specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE
+# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+
+import dask, timeit
+import dask.array as da
+import dask.multiprocessing
+import numpy as np
+
+
+class common_bench:
+    sx, sy = 320000, 1000
+    cx, cy = 10000,  1000
+
+class dask_bench(common_bench):
+    def setup(self):
+        self.x = da.random.random((self.sx, self.sy), chunks=(self.cx, self.cy))
+
+    def _bench(self, get):
+        q, r = da.linalg.qr(self.x)
+        test = da.all(da.isclose(self.x, q.dot(r)))
+        test.compute(get=get)
+
+    def time_threaded(self):
+        self._bench(dask.threaded.get)
+
+    def time_multiproc(self):
+        self._bench(dask.multiprocessing.get)
+
+
+class numpy_bench(common_bench):
+    def setup(self):
+        self.x = np.random.random((self.sx, self.sy))
+
+    def time_pure(self):
+        q, r = np.linalg.qr(self.x)
+        test = np.allclose(self.x, q.dot(r))
+
+print("Numpy  ", timeit.repeat('b.time_pure()', 'from __main__ import numpy_bench as B; b=B();b.setup()', number=1, repeat=3))
+print("Dask-MT", timeit.repeat('b.time_threaded()', 'from __main__ import dask_bench as B; b=B();b.setup()', number=1, repeat=3))
+#print("Dask-MP", timeit.repeat('b.time_multiproc()', 'from __main__ import dask_bench as B; b=B();b.setup()', number=1, repeat=3))