Chinese Custom Tokenizers

Users
, ,
1

this is my code

package main

import (
	"github.com/yanyiwu/gojieba"
)

func Tokenizer() interface{} { return ZHTokenizer{} }

type ZHTokenizer struct{}

func (ZHTokenizer) Name() string     { return "zh" }
func (ZHTokenizer) Type() string     { return "string" }
func (ZHTokenizer) Identifier() byte { return 0x88 }

func (t ZHTokenizer) Tokens(value interface{}) ([]string, error) {
	x := gojieba.NewJieba()
	defer x.Free()
	var toks []string
	toks = x.CutForSearch(value.(string), true)
	return toks, nil
}

When I execute the second query, The docker container aborts.
query:

{
  q(func: allof(<简介>, zh, "嫦娥")) {
    name
    <简介>
  }
}

Some logs:

fatal error: unexpected signal during runtime execution
[signal SIGSEGV: segmentation violation code=0x1 addr=0x74 pc=0x7f64324fd139]


runtime stack:
runtime.throw(0x1bcbca5, 0x2a)
	/usr/local/go/src/runtime/panic.go:1116 +0x72
runtime.sigpanic()
	/usr/local/go/src/runtime/signal_unix.go:679 +0x46a


goroutine 243 [syscall]:
runtime.cgocall(0x7f64324f0140, 0xc00012cb28, 0x7f6432470cfa)
	/usr/local/go/src/runtime/cgocall.go:133 +0x61 fp=0xc00012caf8 sp=0xc00012cac0 pc=0x7f6432468521
github.com/yanyiwu/gojieba._Cfunc_CutForSearch(0x7f6424016390, 0x7f6424016c70, 0x1, 0x0)
	_cgo_gotypes.go:129 +0x53 fp=0xc00012cb28 sp=0xc00012caf8 pc=0x7f64324eca13
github.com/yanyiwu/gojieba.(*Jieba).CutForSearch.func2(0xc000617ee0, 0x7f6424016c70, 0x1, 0x1961000)
	/usr/local/gocode/src/github.com/yanyiwu/gojieba/jieba.go:80 +0x6a fp=0xc00012cb60 sp=0xc00012cb28 pc=0x7f64324eeeba
github.com/yanyiwu/gojieba.(*Jieba).CutForSearch(0xc000617ee0, 0xc02aabc300, 0x2c3, 0xc00012cc01, 0x0, 0x0, 0x0)
	/usr/local/gocode/src/github.com/yanyiwu/gojieba/jieba.go:80 +0xa0 fp=0xc00012cbb8 sp=0xc00012cb60 pc=0x7f64324ed830
plugin/unnamed-1b6668cd2ea8eeb9f037423be800e06b59b781e0.ZHTokenizer.Tokens(0x19615c0, 0xc02f4c6e50, 0x0, 0x0, 0x0, 0x0, 0x0)
	/usr/local/gocode/src/dgraph/_customtok/zh/main.go:36 +0xa8 fp=0xc00012cc18 sp=0xc00012cbb8 pc=0x7f64324efd98
plugin/unnamed-1b6668cd2ea8eeb9f037423be800e06b59b781e0.(*ZHTokenizer).Tokens(0x2bcf318, 0x19615c0, 0xc02f4c6e50, 0x9f91de, 0xc059cd6390, 0xc059cd7ec6, 0x9, 0x19615c0)
	<autogenerated>:1 +0x48 fp=0xc00012cc60 sp=0xc00012cc18 pc=0x7f64324eff88
github.com/dgraph-io/dgraph/tok.(*CustomTokenizer).Tokens(0xc000142170, 0x19615c0, 0xc02f4c6e50, 0x1db3440, 0xc0006b7930, 0xc00003e780, 0xc00012cd38, 0x13c0d02)
	<autogenerated>:1 +0x54 fp=0xc00012ccb0 sp=0xc00012cc60 pc=0x13b26e4
github.com/dgraph-io/dgraph/tok.BuildTokens(0x19615c0, 0xc02f4c6e50, 0x1e142e0, 0xc000142170, 0x1e142e0, 0xc000142170, 0xc02f4c6e50, 0x0, 0x0)
	/ext-go/1/src/github.com/dgraph-io/dgraph/tok/tok.go:107 +0x63 fp=0xc00012cd48 sp=0xc00012ccb0 pc=0x13add83
github.com/dgraph-io/dgraph/posting.indexTokens(0x1e0c1c0, 0xc059b6bda0, 0xc00012d018, 0x2, 0x1e146a0, 0x4, 0x1, 0x2)
	/ext-go/1/src/github.com/dgraph-io/dgraph/posting/index.go:75 +0x2ce fp=0xc00012ce38 sp=0xc00012cd48 pc=0x14ee6ce
github.com/dgraph-io/dgraph/posting.(*Txn).addIndexMutations(0xc02aaacd00, 0x1e0c1c0, 0xc059b6bda0, 0xc00012d018, 0x6, 0xc02a867b20)
	/ext-go/1/src/github.com/dgraph-io/dgraph/posting/index.go:97 +0x118 fp=0xc00012ced0 sp=0xc00012ce38 pc=0x14eeab8
github.com/dgraph-io/dgraph/posting.(*List).AddMutationWithIndex(0xc02aa9d560, 0x1e0c1c0, 0xc059b6bda0, 0xc02aaba140, 0xc02aaacd00, 0x0, 0x6)
	/ext-go/1/src/github.com/dgraph-io/dgraph/posting/index.go:506 +0x40b fp=0xc00012d0a8 sp=0xc00012ced0 pc=0x14f151b
github.com/dgraph-io/dgraph/worker.runMutation(0x1e0c100, 0xc02aa9f5c0, 0xc02aaba140, 0xc02aaacd00, 0x0, 0x0)
	/ext-go/1/src/github.com/dgraph-io/dgraph/worker/mutation.go:118 +0x3e2 fp=0xc00012d2d0 sp=0xc00012d0a8 pc=0x161a6f2
github.com/dgraph-io/dgraph/worker.(*node).applyMutations.func3(0xc02a8677e0, 0x3, 0x4, 0x0, 0x1bc69f2)
	/ext-go/1/src/github.com/dgraph-io/dgraph/worker/draft.go:464 +0xb1 fp=0xc00012d380 sp=0xc00012d2d0 pc=0x164c861
github.com/dgraph-io/dgraph/worker.(*node).applyMutations(0xc0002c2540, 0x1e0c100, 0xc02aa9f5c0, 0xc02aab8120, 0x0, 0x0)
	/ext-go/1/src/github.com/dgraph-io/dgraph/worker/draft.go:483 +0xb85 fp=0xc00012d7c0 sp=0xc00012d380 pc=0x1602575
github.com/dgraph-io/dgraph/worker.(*node).applyCommitted(0xc0002c2540, 0xc02aab8120, 0x0, 0x0)
	/ext-go/1/src/github.com/dgraph-io/dgraph/worker/draft.go:513 +0x11d7 fp=0xc00012db98 sp=0xc00012d7c0 pc=0x16041f7
github.com/dgraph-io/dgraph/worker.(*node).processApplyCh.func1(0xc059588e90, 0x1, 0x1)
	/ext-go/1/src/github.com/dgraph-io/dgraph/worker/draft.go:664 +0x1b0 fp=0xc00012dd48 sp=0xc00012db98 pc=0x164cc70
github.com/dgraph-io/dgraph/worker.(*node).processApplyCh(0xc0002c2540)
	/ext-go/1/src/github.com/dgraph-io/dgraph/worker/draft.go:705 +0x246 fp=0xc00012dfd8 sp=0xc00012dd48 pc=0x1604836
runtime.goexit()
	/usr/local/go/src/runtime/asm_amd64.s:1373 +0x1 fp=0xc00012dfe0 sp=0xc00012dfd8 pc=0xa4eb21
created by github.com/dgraph-io/dgraph/worker.(*node).InitAndStartNode
	/ext-go/1/src/github.com/dgraph-io/dgraph/worker/draft.go:1650 +0x4c6
3

_Cfunc_CutForSearch 有异常啊
单元测试能跑吗?
另外,这个查询也不对好像.
这个zh,好像是加到schema的索引里.

{
  q(func: allof(<简介>, zh, "嫦娥")) {
    name
    <简介>
  }
}
4

单元测试能跑，Dgraph启动后第一次查询能返回结果，第二次查询就会出异常。同样的如果是设置schema也是第一次可以，第二次会异常，容器停止运行。“zh”是我定义的自定义索引的名称也是分词器的注册名。在研究是不是因为加载c++库的问题，打算用纯go实现的分词再封装一个分词器。不知道有没有介绍呢？

5

我的目的是在Dgraph里创建中文的全文检索索引，而不是自带的谷歌全文索引。这样我就能引入领域字典提高检索的准确率。

6

可能与cgo有关系,这一部分的内存应该不是go管理的.