Golang bfs-store-needle.Needle类(方法)实例源码

下面列出了Golang bfs-store-needle.Needle 类(方法)源码代码实例，从而了解它的用法。

作者：yongleho    项目：bf    
func (s *Server) get(wr http.ResponseWriter, r *http.Request) {
	var (
		v                *volume.Volume
		n                *needle.Needle
		err              error
		vid, key, cookie int64
		ret              = http.StatusOK
		params           = r.URL.Query()
		now              = time.Now()
	)
	if r.Method != "GET" && r.Method != "HEAD" {
		ret = http.StatusMethodNotAllowed
		http.Error(wr, "method not allowed", ret)
		return
	}
	defer HttpGetWriter(r, wr, now, &err, &ret)
	if vid, err = strconv.ParseInt(params.Get("vid"), 10, 32); err != nil {
		log.Errorf("strconv.ParseInt(\"%s\") error(%v)", params.Get("vid"), err)
		ret = http.StatusBadRequest
		return
	}
	if key, err = strconv.ParseInt(params.Get("key"), 10, 64); err != nil {
		log.Errorf("strconv.ParseInt(\"%s\") error(%v)", params.Get("key"), err)
		ret = http.StatusBadRequest
		return
	}
	if cookie, err = strconv.ParseInt(params.Get("cookie"), 10, 32); err != nil {
		log.Errorf("strconv.ParseInt(\"%s\") error(%v)", params.Get("cookie"), err)
		ret = http.StatusBadRequest
		return
	}
	n = s.store.Needle()
	n.Key = key
	n.Cookie = int32(cookie)
	if v = s.store.Volumes[int32(vid)]; v != nil {
		if err = v.Get(n); err != nil {
			if err == errors.ErrNeedleDeleted || err == errors.ErrNeedleNotExist {
				ret = http.StatusNotFound
			} else {
				ret = http.StatusInternalServerError
			}
		}
	} else {
		ret = http.StatusNotFound
		err = errors.ErrVolumeNotExist
	}
	if err == nil {
		if r.Method == "GET" {
			if _, err = wr.Write(n.Data); err != nil {
				log.Errorf("wr.Write() error(%v)", err)
				ret = http.StatusInternalServerError
			}
		}
		if log.V(1) {
			log.Infof("get a needle: %v", n)
		}
	}
	s.store.FreeNeedle(n)
	return
}
作者：yongleho    项目：bf    
func compareTestNeedle(t *testing.T, key int64, cookie int32, flag byte, n *needle.Needle, data []byte) (err error) {
	if err = n.Parse(); err != nil {
		t.Error(err)
		return
	}
	if !bytes.Equal(n.Data, data) {
		err = fmt.Errorf("data: %s not match", n.Data)
		t.Error(err)
		return
	}
	if n.Cookie != cookie {
		err = fmt.Errorf("cookie: %d not match", n.Cookie)
		t.Error(err)
		return
	}
	if n.Key != key {
		err = fmt.Errorf("key: %d not match", n.Key)
		t.Error(err)
		return
	}
	if n.Flag != flag {
		err = fmt.Errorf("flag: %d not match", n.Flag)
		t.Error(err)
		return
	}
	if n.Size != int32(len(data)) {
		err = fmt.Errorf("size: %d not match", n.Size)
		t.Error(err)
		return
	}
	return
}
作者：yongleho    项目：bf    
// ReadAt read a needle by specified offset, before call it, must set needle
// TotalSize.
func (b *SuperBlock) ReadAt(offset uint32, n *needle.Needle) (err error) {
	if b.LastErr != nil {
		return b.LastErr
	}
	err = n.ReadAt(offset, b.r)
	b.LastErr = err
	return
}
作者：yongleho    项目：bf    
// WriteAt write a needle by specified offset;
func (b *SuperBlock) WriteAt(offset uint32, n *needle.Needle) (err error) {
	if b.LastErr != nil {
		return b.LastErr
	}
	err = n.WriteAt(offset, b.w)
	b.LastErr = err
	return
}
作者：yongleho    项目：bf    
func (s *Server) upload(wr http.ResponseWriter, r *http.Request) {
	var (
		vid    int64
		key    int64
		cookie int64
		size   int64
		err    error
		str    string
		v      *volume.Volume
		n      *needle.Needle
		file   multipart.File
		res    = map[string]interface{}{}
	)
	if r.Method != "POST" {
		http.Error(wr, "method not allowed", http.StatusMethodNotAllowed)
		return
	}
	defer HttpPostWriter(r, wr, time.Now(), &err, res)
	if err = checkContentLength(r, s.conf.NeedleMaxSize); err != nil {
		return
	}
	str = r.FormValue("vid")
	if vid, err = strconv.ParseInt(str, 10, 32); err != nil {
		log.Errorf("strconv.ParseInt(\"%s\") error(%v)", str, err)
		err = errors.ErrParam
		return
	}
	str = r.FormValue("key")
	if key, err = strconv.ParseInt(str, 10, 64); err != nil {
		log.Errorf("strconv.ParseInt(\"%s\") error(%v)", str, err)
		err = errors.ErrParam
		return
	}
	str = r.FormValue("cookie")
	if cookie, err = strconv.ParseInt(str, 10, 32); err != nil {
		log.Errorf("strconv.ParseInt(\"%s\") error(%v)", str, err)
		err = errors.ErrParam
		return
	}
	if file, _, err = r.FormFile("file"); err != nil {
		log.Errorf("r.FormFile() error(%v)", err)
		err = errors.ErrInternal
		return
	}
	if size, err = checkFileSize(file, s.conf.NeedleMaxSize); err == nil {
		n = s.store.Needle()
		if err = n.WriteFrom(key, int32(cookie), int32(size), file); err == nil {
			if v = s.store.Volumes[int32(vid)]; v != nil {
				err = v.Write(n)
			} else {
				err = errors.ErrVolumeNotExist
			}
		}
		s.store.FreeNeedle(n)
	}
	file.Close()
	return
}
作者：Terry-Ma    项目：bf    
// WriteAt write a needle by specified offset;
func (b *SuperBlock) WriteAt(offset uint32, n *needle.Needle) (err error) {
	if b.LastErr != nil {
		return b.LastErr
	}
	if _, err = b.w.WriteAt(n.Buffer(), needle.BlockOffset(offset)); err != nil {
		b.LastErr = err
	}
	return
}
作者：Terry-Ma    项目：bf    
// ReadAt read a needle by specified offset, before call it, must set needle
// TotalSize.
func (b *SuperBlock) ReadAt(n *needle.Needle) (err error) {
	if b.LastErr != nil {
		return b.LastErr
	}
	if _, err = b.r.ReadAt(n.Buffer(), needle.BlockOffset(n.Offset)); err == nil {
		err = n.Parse()
	} else {
		b.LastErr = err
	}
	return
}
作者：yongleho    项目：bf    
func (v *Volume) get(n *needle.Needle) (err error) {
	var (
		ok     bool
		nc     int64
		offset uint32
		size   int32
		key    = n.Key
		now    = time.Now().UnixNano()
	)
	// pread syscall is atomic, so use rlock
	v.lock.RLock()
	if nc, ok = v.needles[key]; ok {
		if offset, size = needle.Cache(nc); offset != needle.CacheDelOffset {
			n.TotalSize = size
			err = v.Block.ReadAt(offset, n)
		} else {
			err = errors.ErrNeedleDeleted
		}
	} else {
		err = errors.ErrNeedleNotExist
	}
	v.lock.RUnlock()
	if err != nil {
		return
	}
	if err = n.Parse(); err != nil {
		return
	}
	if n.Key != key {
		return errors.ErrNeedleKey
	}
	if n.TotalSize != size {
		return errors.ErrNeedleSize
	}
	if log.V(1) {
		log.Infof("get needle key: %d, cookie: %d, offset: %d, size: %d", n.Key, n.Cookie, offset, size)
		log.Infof("%v\n", n)
	}
	// needles map may be out-dated, recheck
	if n.Flag == needle.FlagDel {
		v.lock.Lock()
		v.needles[key] = needle.NewCache(needle.CacheDelOffset, size)
		v.lock.Unlock()
		err = errors.ErrNeedleDeleted
	} else {
		atomic.AddUint64(&v.Stats.TotalGetProcessed, 1)
		atomic.AddUint64(&v.Stats.TotalReadBytes, uint64(size))
		atomic.AddUint64(&v.Stats.TotalGetDelay, uint64(time.Now().UnixNano()-now))
	}
	return
}
作者：Terry-Ma    项目：bf    
// Write write needle to the block.
func (b *SuperBlock) Write(n *needle.Needle) (err error) {
	if b.LastErr != nil {
		return b.LastErr
	}
	if _maxOffset-n.IncrOffset < b.Offset {
		err = errors.ErrSuperBlockNoSpace
		return
	}
	if _, err = b.w.Write(n.Buffer()); err == nil {
		err = b.flush(false)
	} else {
		b.LastErr = err
		return
	}
	b.Offset += n.IncrOffset
	b.Size += int64(n.TotalSize)
	return
}
作者：yongleho    项目：bf    
// Probe probe a needle by key.
func (v *Volume) Probe(n *needle.Needle) (err error) {
	var key int64
	v.lock.RLock()
	// get a rand key
	for key, _ = range v.needles {
		break
	}
	v.lock.RUnlock()
	n.Key = key
	if err = v.get(n); err != nil {
		return
	}
	return
}
作者：Terry-Ma    项目：bf    
func TestVolume(t *testing.T) {
	var (
		v     *Volume
		n     *needle.Needle
		err   error
		data  = []byte("test")
		bfile = "../test/test1"
		ifile = "../test/test1.idx"
		ns    = needle.NewNeedles(3)
		buf   = &bytes.Buffer{}
	)
	os.Remove(bfile)
	os.Remove(ifile)
	defer os.Remove(bfile)
	defer os.Remove(ifile)
	if v, err = NewVolume(1, bfile, ifile, _c); err != nil {
		t.Errorf("NewVolume() error(%v)", err)
		t.FailNow()
	}
	v.Close()
	// test open
	if err = v.Open(); err != nil {
		t.Errorf("Open() error(%v)", err)
		t.FailNow()
	}
	defer v.Close()
	// test write
	if _, err = buf.Write(data); err != nil {
		t.Errorf("buf.Write() error(%v)", err)
		t.FailNow()
	}
	n = needle.NewWriter(1, 1, 4)
	defer n.Close()
	if err = n.ReadFrom(buf); err != nil {
		t.Errorf("n.Write() error(%v)", err)
		t.FailNow()
	}
	if err = v.Write(n); err != nil {
		t.Errorf("Add() error(%v)", err)
		t.FailNow()
	}
	if _, err = buf.Write(data); err != nil {
		t.Errorf("buf.Write() error(%v)", err)
		t.FailNow()
	}
	n = needle.NewWriter(2, 2, 4)
	defer n.Close()
	if err = n.ReadFrom(buf); err != nil {
		t.Errorf("n.Write() error(%v)", err)
		t.FailNow()
	}
	if err = v.Write(n); err != nil {
		t.Errorf("Add() error(%v)", err)
		t.FailNow()
	}
	if _, err = buf.Write(data); err != nil {
		t.Errorf("buf.Write() error(%v)", err)
		t.FailNow()
	}
	n = needle.NewWriter(3, 3, 4)
	defer n.Close()
	if err = n.ReadFrom(buf); err != nil {
		t.Errorf("n.Write() error(%v)", err)
		t.FailNow()
	}
	if err = v.Write(n); err != nil {
		t.Errorf("Add() error(%v)", err)
		t.FailNow()
	}
	if _, err = buf.Write(data); err != nil {
		t.Errorf("buf.Write() error(%v)", err)
		t.FailNow()
	}
	if err = ns.ReadFrom(4, 4, 4, buf); err != nil {
		t.Errorf("ns.Write() error(%v)", err)
		t.FailNow()
	}
	if _, err = buf.Write(data); err != nil {
		t.Errorf("buf.Write() error(%v)", err)
		t.FailNow()
	}
	if err = ns.ReadFrom(5, 5, 4, buf); err != nil {
		t.Errorf("ns.Write() error(%v)", err)
		t.FailNow()
	}
	if _, err = buf.Write(data); err != nil {
		t.Errorf("buf.Write() error(%v)", err)
		t.FailNow()
	}
	if err = ns.ReadFrom(6, 6, 4, buf); err != nil {
		t.Errorf("ns.Write() error(%v)", err)
		t.FailNow()
	}
	if err = v.Writes(ns); err != nil {
		t.Errorf("Write() error(%v)", err)
		t.FailNow()
	}
	if err = v.Delete(3); err != nil {
		t.Errorf("Del error(%v)", err)
		t.FailNow()
//.........这里部分代码省略.........
作者：Terry-Ma    项目：bf    
// Scan scan a block file.
func (b *SuperBlock) Scan(r *os.File, offset uint32, fn func(*needle.Needle, uint32, uint32) error) (err error) {
	var (
		so, eo uint32
		bso    int64
		fi     os.FileInfo
		fd     = r.Fd()
		n      = new(needle.Needle)
		rd     = bufio.NewReaderSize(r, b.conf.Block.BufferSize)
	)
	if offset == 0 {
		offset = needle.NeedleOffset(_headerOffset)
	}
	so, eo = offset, offset
	bso = needle.BlockOffset(so)
	// advise sequential read
	if fi, err = r.Stat(); err != nil {
		log.Errorf("block: %s Stat() error(%v)", b.File)
		return
	}
	if err = myos.Fadvise(fd, bso, fi.Size(), myos.POSIX_FADV_SEQUENTIAL); err != nil {
		log.Errorf("block: %s Fadvise() error(%v)", b.File)
		return
	}
	log.Infof("scan block: %s from offset: %d", b.File, offset)
	if _, err = r.Seek(bso, os.SEEK_SET); err != nil {
		log.Errorf("block: %s Seek() error(%v)", b.File)
		return
	}
	for {
		if err = n.ParseFrom(rd); err != nil {
			if err != io.EOF {
				log.Errorf("block: parse needle from offset: %d:%d error(%v)", so, eo, err)
			}
			break
		}
		if n.TotalSize > int32(b.conf.BlockMaxSize) {
			log.Errorf("scan block: %s error(%v)", n, errors.ErrNeedleSize)
			err = errors.ErrNeedleSize
			break
		}
		if log.V(1) {
			log.Info(n.String())
		}
		eo += n.IncrOffset
		if err = fn(n, so, eo); err != nil {
			log.Errorf("block: callback from offset: %d:%d error(%v)", so, eo, err)
			break
		}
		so = eo
	}
	if err == io.EOF {
		// advise no need page cache
		if err = myos.Fadvise(fd, bso, needle.BlockOffset(eo-so), myos.POSIX_FADV_DONTNEED); err != nil {
			log.Errorf("block: %s Fadvise() error(%v)", b.File)
			return
		}
		log.Infof("scan block: %s to offset: %d [ok]", b.File, eo)
		err = nil
	} else {
		log.Infof("scan block: %s to offset: %d error(%v) [failed]", b.File, eo, err)
	}
	return
}
作者：Terry-Ma    项目：bf    
func TestSuperBlock(t *testing.T) {
	var (
		b                  *SuperBlock
		n                  *needle.Needle
		offset, v2, v3, v4 uint32
		err                error
		buf                = &bytes.Buffer{}
		needles            = make(map[int64]int64)
		data               = []byte("test")
		file               = "../test/test.block"
		ifile              = "../test/test.idx"
		//indexer *Indexer
	)
	os.Remove(file)
	os.Remove(ifile)
	defer os.Remove(file)
	defer os.Remove(ifile)
	// test new block file
	if b, err = NewSuperBlock(file, testConf); err != nil {
		t.Errorf("NewSuperBlock(\"%s\") error(%v)", file, err)
		t.FailNow()
	}
	b.Close()
	// test parse block file
	if b, err = NewSuperBlock(file, testConf); err != nil {
		t.Errorf("NewSuperBlock(\"%s\") error(%v)", file, err)
		t.FailNow()
	}
	b.Close()
	// test open
	if err = b.Open(); err != nil {
		t.Errorf("Open() error(%v)", err)
		t.FailNow()
	}
	defer b.Close()
	// test write
	if _, err = buf.Write(data); err != nil {
		t.Errorf("buf.Write() error(%v)", err)
		t.FailNow()
	}
	n = needle.NewWriter(1, 1, 4)
	defer n.Close()
	n.ReadFrom(buf)
	if err = b.Write(n); err != nil {
		t.Errorf("b.Write() error(%v)", err)
		t.FailNow()
	}
	if err = compareTestOffset(b, n, needle.NeedleOffset(int64(_headerSize))); err != nil {
		t.Errorf("compareTestOffset() error(%v)", err)
		t.FailNow()
	}
	offset = b.Offset
	v2 = b.Offset
	// test get
	n.Offset = 1
	if err = b.ReadAt(n); err != nil {
		t.Errorf("b.ReadAt() error(%v)", err)
		t.FailNow()
	}
	if err = compareTestNeedle(t, 1, 1, needle.FlagOK, n, data); err != nil {
		t.Errorf("compareTestNeedle() error(%v)", err)
		t.FailNow()
	}
	// test write
	if _, err = buf.Write(data); err != nil {
		t.Errorf("buf.Write() error(%v)", err)
		t.FailNow()
	}
	n = needle.NewWriter(2, 2, 4)
	defer n.Close()
	n.ReadFrom(buf)
	if err = b.Write(n); err != nil {
		t.Errorf("b.Write() error(%v)", err)
		t.FailNow()
	}
	if err = compareTestOffset(b, n, offset); err != nil {
		t.Errorf("compareTestOffset() error(%v)", err)
		t.FailNow()
	}
	offset = b.Offset
	v3 = b.Offset
	n.Offset = 6
	if err = b.ReadAt(n); err != nil {
		t.Errorf("b.ReadAt() error(%v)", err)
		t.FailNow()
	}
	if err = compareTestNeedle(t, 2, 2, needle.FlagOK, n, data); err != nil {
		t.Error("compareTestNeedle(2)")
		t.FailNow()
	}
	// test write
	if _, err = buf.Write(data); err != nil {
		t.Errorf("buf.Write() error(%v)", err)
		t.FailNow()
	}
	n = needle.NewWriter(3, 3, 4)
	defer n.Close()
	n.ReadFrom(buf)
	if err = b.Write(n); err != nil {
		t.Errorf("b.Write() error(%v)", err)
//.........这里部分代码省略.........
作者：Terry-Ma    项目：bf    
func TestStore(t *testing.T) {
	var (
		s   *Store
		z   *zk.Zookeeper
		v   *volume.Volume
		n   *needle.Needle
		err error
		buf = &bytes.Buffer{}
	)
	os.Remove(testConf.Store.VolumeIndex)
	os.Remove(testConf.Store.FreeVolumeIndex)
	os.Remove("./test/_free_block_1")
	os.Remove("./test/_free_block_1.idx")
	os.Remove("./test/_free_block_2")
	os.Remove("./test/_free_block_2.idx")
	os.Remove("./test/_free_block_3")
	os.Remove("./test/_free_block_3.idx")
	os.Remove("./test/1_0")
	os.Remove("./test/1_0.idx")
	os.Remove("./test/1_1")
	os.Remove("./test/1_1.idx")
	os.Remove("./test/block_store_1")
	os.Remove("./test/block_store_1.idx")
	defer os.Remove(testConf.Store.VolumeIndex)
	defer os.Remove(testConf.Store.FreeVolumeIndex)
	defer os.Remove("./test/_free_block_1")
	defer os.Remove("./test/_free_block_1.idx")
	defer os.Remove("./test/_free_block_2")
	defer os.Remove("./test/_free_block_2.idx")
	defer os.Remove("./test/_free_block_3")
	defer os.Remove("./test/_free_block_3.idx")
	defer os.Remove("./test/1_0")
	defer os.Remove("./test/1_0.idx")
	defer os.Remove("./test/1_1")
	defer os.Remove("./test/1_1.idx")
	defer os.Remove("./test/block_store_1")
	defer os.Remove("./test/block_store_1.idx")
	if z, err = zk.NewZookeeper(testConf); err != nil {
		t.Errorf("NewZookeeper() error(%v)", err)
		t.FailNow()
	}
	defer z.Close()
	z.DelVolume(1)
	z.DelVolume(2)
	z.DelVolume(3)
	defer z.DelVolume(1)
	defer z.DelVolume(2)
	defer z.DelVolume(3)
	if s, err = NewStore(testConf); err != nil {
		t.Errorf("NewStore() error(%v)", err)
		t.FailNow()
	}
	defer s.Close()
	if _, err = s.AddFreeVolume(2, "./test", "./test"); err != nil {
		t.Errorf("s.AddFreeVolume() error(%v)", err)
		t.FailNow()
	}
	if v, err = s.AddVolume(1); err != nil {
		t.Errorf("AddVolume() error(%v)", err)
		t.FailNow()
	}
	if v = s.Volumes[1]; v == nil {
		t.Error("Volume(1) not exist")
		t.FailNow()
	}
	buf.WriteString("test")
	n = needle.NewWriter(1, 1, 4)
	if err = n.ReadFrom(buf); err != nil {
		t.Errorf("n.ReadFrom() error(%v)", err)
		t.FailNow()
	}
	if err = v.Write(n); err != nil {
		t.Errorf("v.Add(1) error(%v)", err)
		t.FailNow()
	}
	if _, err = v.Read(1, 1); err != nil {
		t.Errorf("v.WriteTo(1) error(%v)", err)
		t.FailNow()
	}
	if err = s.BulkVolume(2, "./test/block_store_1", "./test/block_store_1.idx"); err != nil {
		t.Errorf("Bulk(1) error(%v)", err)
		t.FailNow()
	}
	if v = s.Volumes[2]; v == nil {
		t.Error("Volume(2) not exist")
		t.FailNow()
	}
	buf.WriteString("test")
	n = needle.NewWriter(1, 1, 4)
	if err = n.ReadFrom(buf); err != nil {
		t.Errorf("n.ReadFrom() error(%v)", err)
		t.FailNow()
	}
	if err = v.Write(n); err != nil {
		t.Errorf("v.Add() error(%v)", err)
		t.FailNow()
	}
	if n, err = v.Read(1, 1); err != nil {
		t.Errorf("v.WriteTo(1) error(%v)", err)
		t.FailNow()
//.........这里部分代码省略.........

提
问题

写
面经

写
文章

微信
公众号

扫码关注公众号