v1.1.4

mayabot · Sep 8, 2018 · 4b69773 · 4b69773
1 parent 95157be
commit 4b69773
Show file tree

Hide file tree

Showing 15 changed files with 25 additions and 1,108 deletions.
diff --git a/build.gradle b/build.gradle
@@ -8,7 +8,7 @@ plugins {
 
 
 description = 'FastText的java版本实现，兼容facebook发布的原生预训练模型。'
-version = "1.1.3"
+version = "1.1.4"
 //.BUILD-SNAPSHOT
 
 group = "com.mayabot"
@@ -33,6 +33,7 @@ dependencies {
     compile "org.jetbrains.kotlin:kotlin-stdlib-jdk8"
 
     compile group: 'com.carrotsearch', name: 'hppc', version: '0.7.3'
+    compile 'com.mayabot:maya-simple-blas:1.0.0'
     compile group: 'com.google.guava', name: 'guava', version: "19.0"
 
     testCompile 'junit:junit:4.12'

diff --git a/src/main/java/com/mayabot/mynlp/fasttext/Args.kt b/src/main/java/com/mayabot/mynlp/fasttext/Args.kt
@@ -1,6 +1,7 @@
 package com.mayabot.mynlp.fasttext
 
 
+import com.mayabot.blas.AutoDataInput
 import java.io.IOException
 import java.nio.channels.FileChannel
 

diff --git a/src/main/java/com/mayabot/mynlp/fasttext/AutoDataInput.kt b/src/main/java/com/mayabot/mynlp/fasttext/AutoDataInput.kt
diff --git a/src/main/java/com/mayabot/mynlp/fasttext/ByteUtils.java b/src/main/java/com/mayabot/mynlp/fasttext/ByteUtils.java
diff --git a/src/main/java/com/mayabot/mynlp/fasttext/Dictionary.kt b/src/main/java/com/mayabot/mynlp/fasttext/Dictionary.kt
@@ -8,10 +8,9 @@ import com.google.common.base.CharMatcher
 import com.google.common.base.Splitter
 import java.io.File
 import java.io.IOException
-import java.math.BigInteger
 import java.nio.ByteBuffer
-import java.nio.ByteOrder
 import java.nio.channels.FileChannel
+import com.mayabot.blas.*
 import java.util.*
 import com.google.common.primitives.UnsignedLong
 
@@ -280,8 +279,8 @@ class Dictionary(private val args: Args) {
         }
     }
 
-    private fun pushHash(hashes: IntArrayList, id: Int) {
-        var id = id
+    private fun pushHash(hashes: IntArrayList, id_: Int) {
+        var id = id_
         if (pruneidxSize == 0L || id < 0) return
 
         if (pruneidxSize > 0) {

diff --git a/src/main/java/com/mayabot/mynlp/fasttext/FastText.kt b/src/main/java/com/mayabot/mynlp/fasttext/FastText.kt
@@ -9,8 +9,8 @@ import com.google.common.collect.Lists
 import com.google.common.collect.Sets
 import com.google.common.io.Files
 import com.google.common.primitives.Floats
-import com.mayabot.mynlp.fasttext.matrix.*
-import com.mayabot.mynlp.fasttext.matrix.Vector
+import com.mayabot.blas.*
+import com.mayabot.blas.Vector
 import fasttext.QMatrix
 import java.io.DataInputStream
 import java.io.File
@@ -360,6 +360,9 @@ class FastText(internal val args: Args,
             return LoadFastTextFromClangModel.loadCModel(modelPath)
         }
 
+        private fun File.openAutoDataInput() = AutoDataInput.open(this)
+
+
         /**
          * 加载java程序保存的文件模型.
          * path应该是一个目录，下面保存各个细节的文件
@@ -572,6 +575,16 @@ class Model(private val inputMatrix: FloatMatrix
         }
     }
 
+    private fun matrixMulVector(matrix: QMatrix, v: Vector, target: MutableVector) {
+        checkArgument(matrix.m == target.length())
+        checkArgument(matrix.n == v.length())
+
+        val m_ = matrix.m
+        for (i in 0 until m_) {
+            target[i] = matrix.dotRow(v,i)
+        }
+    }
+
 }
 
 /**

diff --git a/src/main/java/com/mayabot/mynlp/fasttext/FastTextTrain.kt b/src/main/java/com/mayabot/mynlp/fasttext/FastTextTrain.kt
@@ -8,7 +8,7 @@ import com.google.common.collect.Lists
 import com.google.common.io.Files
 import com.google.common.primitives.Ints
 import com.google.common.util.concurrent.AtomicDouble
-import com.mayabot.mynlp.fasttext.matrix.*
+import com.mayabot.blas.*
 import fasttext.QMatrix
 import java.io.*
 import java.util.concurrent.atomic.AtomicLong

diff --git a/src/main/java/com/mayabot/mynlp/fasttext/IOUtils.kt b/src/main/java/com/mayabot/mynlp/fasttext/IOUtils.kt
@@ -1,5 +1,6 @@
 package com.mayabot.mynlp.fasttext
 
+import com.mayabot.blas.ByteUtils
 import java.io.IOException
 import java.nio.ByteBuffer
 import java.nio.channels.FileChannel

diff --git a/src/main/java/com/mayabot/mynlp/fasttext/Matrix.kt b/src/main/java/com/mayabot/mynlp/fasttext/Matrix.kt
diff --git a/src/main/java/com/mayabot/mynlp/fasttext/ProductQuantizer.kt b/src/main/java/com/mayabot/mynlp/fasttext/ProductQuantizer.kt
@@ -3,10 +3,8 @@ package fasttext
 
 import com.google.common.base.Preconditions
 import com.mayabot.mynlp.fasttext.*
-import com.mayabot.mynlp.fasttext.matrix.FloatMatrix
-import com.mayabot.mynlp.fasttext.matrix.MutableFloatMatrix
-import com.mayabot.mynlp.fasttext.matrix.MutableVector
-import com.mayabot.mynlp.fasttext.matrix.Vector
+import com.mayabot.blas.*
+import com.mayabot.blas.Vector
 import java.io.IOException
 import java.nio.ByteBuffer
 import java.nio.channels.FileChannel