Golang compress-zlib.NewReader类(方法)实例源码

func testCompress(t *testing.T) {
	assert := assert.New(t)
	before := `
2e 00 00 00 03 73 65 6c    65 63 74 20 22 30 31 32    .....select "012
33 34 35 36 37 38 39 30    31 32 33 34 35 36 37 38    3456789012345678
39 30 31 32 33 34 35 36    37 38 39 30 31 32 33 34    9012345678901234
35 22                                                 5"
`
	after := `
22 00 00 00 32 00 00 78    9c d3 63 60 60 60 2e 4e    "...2..x..c....N
cd 49 4d 2e 51 50 32 30    34 32 36 31 35 33 b7 b0    .IM.QP20426153..
c4 cd 52 02 00 0c d1 0a    6c                         ..R.....l
`
	_, _ = before, after
	{
		data := bytes.NewBuffer(DecodeDump(after))
		data.Read(make([]byte, 7))
		r, err := zlib.NewReader(data)
		assert.NoError(err)
		b, err := ioutil.ReadAll(r)
		assert.NoError(err)
		assert.EqualValues(DecodeDump(before), b)
	}
	{
		var data bytes.Buffer
		w, err := zlib.NewWriterLevel(&data, zlib.BestCompression)
		assert.NoError(err)
		h, err := hex.DecodeString(strings.Replace("22 00 00 00 32 00 00", " ", "", -1))
		assert.NoError(err)
		data.Write(h)
		w.Write(DecodeDump(before))
		w.Close() // Important

		fmt.Println(hex.Dump(data.Bytes()))
		fmt.Println(hex.Dump(DecodeDump(after)))
		//		fmt.Println(hex.Dump(DecodeDump(before)))

		// 不等,因为会刷一个 00 00 FF FF 的 deflate 块边界
		//		assert.EqualValues(DecodeDump(after), data.Bytes())
		data.Read(make([]byte, 7))
		fmt.Println(hex.Dump(data.Bytes()))
		r, err := zlib.NewReader(&data)
		assert.NoError(err)
		b, err := ioutil.ReadAll(r)
		assert.NoError(err)
		r.Close()
		assert.EqualValues(DecodeDump(before), b)
	}

}

func load(r redis.AsyncClient, k string, w http.ResponseWriter) (obj interface{}) {
	f, rerr := r.Get(k)
	if rerr != nil {
		panic(rerr)
	}
	val, rerr, timeout := f.TryGet(50000000000)
	if rerr != nil {
		panic(rerr)
	}
	if timeout {
		loadtimeout++
		log.Println("load timeout! count: ", loadtimeout)
		fmt.Fprintf(w, "Save failed for %s", key)
		return
	}
	zr, err := zlib.NewReader(bytes.NewReader(val))
	if err != nil {
		log.Fatal("Failed to create zlib reader with error: ", err)
	}
	defer zr.Close()
	jd := json.NewDecoder(zr)
	err = jd.Decode(&obj)
	if err != nil {
		log.Fatal("Failed to decode json with error: ", err)
	}
	return
}

func (this *PacketGeneric) Decompress() (err error) {
	if !this.compressed {
		return
	}
	buffer := bytes.NewReader(this.Bytes)
	_, err = packet.ReadVarInt(buffer) // compression length
	if err != nil {
		return
	}
	zlibReader, err := zlib.NewReader(buffer)
	if err != nil {
		return
	}
	_, err = packet.ReadVarInt(zlibReader) // id
	if err != nil {
		return
	}
	bytes, err := ioutil.ReadAll(zlibReader)
	if err != nil {
		return
	}
	this.Bytes = bytes
	this.compressed = false
	return
}

func (rs *RedisStorage) GetActionPlan(key string, skipCache bool) (ats *ActionPlan, err error) {
	key = utils.ACTION_PLAN_PREFIX + key
	if !skipCache {
		if x, err := cache2go.Get(key); err == nil {
			return x.(*ActionPlan), nil
		} else {
			return nil, err
		}
	}
	var values []byte
	if values, err = rs.db.Cmd("GET", key).Bytes(); err == nil {
		b := bytes.NewBuffer(values)
		r, err := zlib.NewReader(b)
		if err != nil {
			return nil, err
		}
		out, err := ioutil.ReadAll(r)
		if err != nil {
			return nil, err
		}
		r.Close()
		err = rs.ms.Unmarshal(out, &ats)
		cache2go.Cache(key, ats)
	}
	return
}

func (rs *RedisStorage) GetDestination(key string) (dest *Destination, err error) {
	key = utils.DESTINATION_PREFIX + key
	var values []byte
	if values, err = rs.db.Get(key); len(values) > 0 && err == nil {
		b := bytes.NewBuffer(values)
		r, err := zlib.NewReader(b)
		if err != nil {
			return nil, err
		}
		out, err := ioutil.ReadAll(r)
		if err != nil {
			return nil, err
		}
		r.Close()
		dest = new(Destination)
		err = rs.ms.Unmarshal(out, dest)
		// create optimized structure
		for _, p := range dest.Prefixes {
			cache2go.CachePush(utils.DESTINATION_PREFIX+p, dest.Id)
		}
	} else {
		return nil, errors.New("not found")
	}
	return
}

func newLooseObjectEntry(root string, id SHA1) (*looseObjectEntry, error) {
	s := id.String()
	path := filepath.Join(root, "objects", s[:2], s[2:])

	e := new(looseObjectEntry)
	file, err := os.Open(path)
	if err != nil {
		return nil, err
	}
	e.f = file

	zr, err := zlib.NewReader(file)
	if err != nil {
		file.Close()
		return nil, err
	}
	e.zr = zr

	var bs []byte
	e.br = bufio.NewReader(zr)
	if bs, err = e.br.ReadBytes(' '); err != nil {
		e.Close()
		return nil, err
	}
	e.typ = string(bs[:len(bs)-1])

	if _, err = e.br.ReadBytes(0); err != nil {
		e.Close()
		return nil, err
	}
	return e, nil
}

func readCompressed(r *io.SectionReader, offset int64, s []byte) (int, error) {
	zr, err := zlib.NewReader(io.NewSectionReader(r, offset, r.Size()-offset))
	if err != nil {
		return 0, err
	}
	return io.ReadFull(zr, s)
}

// recv gets a message from the connection.
func (conn *Conn) recv() (s []byte, err error) {
	// A message is:
	// - a uint32 length
	// - a byte boolean for compression
	// - length-5 bytes of data (plain or zlib compressed)
	var buf [5]byte
	_, err = io.ReadFull(conn.r, buf[:])
	if err != nil {
		return nil, err
	}
	length := binary.BigEndian.Uint32(buf[:4])
	isCompressed := buf[4] == 1
	if length >= 32<<20 {
		return nil, errMsgTooLarge
	}

	s = make([]byte, length-5)
	_, err = io.ReadFull(conn.r, s)
	if err != nil {
		return
	}
	if isCompressed {
		zr, err := zlib.NewReader(bytes.NewBuffer(s))
		if err != nil {
			return s, err
		}
		return ioutil.ReadAll(zr)
	}
	return s, nil
}

func main() {
	app := cli.NewApp()
	app.Name = "zlib"
	app.Usage = "A command-line tool for using the zlib compression algorithm."
	app.Action = func(c *cli.Context) {
		var reader io.Reader = os.Stdin
		if c.Bool("decompress") {
			compressorReadCloser, err := zlib.NewReader(reader)
			if err != nil {
				exit(err.Error(), 1)
			}
			if _, err := io.Copy(os.Stdout, compressorReadCloser); err != nil {
				exit(err.Error(), 1)
			}
			compressorReadCloser.Close()
		} else {
			var writer io.Writer = os.Stdout
			compressorWriteCloser := zlib.NewWriter(writer)
			if _, err := io.Copy(compressorWriteCloser, reader); err != nil {
				exit(err.Error(), 1)
			}
			compressorWriteCloser.Close()
		}
	}
	app.Flags = []cli.Flag{
		cli.BoolFlag{
			Name:  "d, decompress",
			Usage: "Decompresses the input instead of compressing the output.",
		},
	}
	app.Run(os.Args)
}

func GetObject(repo *libgit.Repository, sha1 [20]byte) (GitObject, error) {
	objectname := fmt.Sprintf("%s/objects/%x/%x", repo.Path, sha1[0:1], sha1[1:])
	fmt.Printf("File: %s\n", objectname)
	f, err := os.Open(objectname)
	if err != nil {
		panic("Couldn't open object file.")
	}
	defer f.Close()

	uncompressed, err := zlib.NewReader(f)
	if err != nil {
		return nil, err
	}
	b, err := ioutil.ReadAll(uncompressed)
	if err != nil {
		return nil, err
	}
	if string(b[0:5]) == "blob " {
		var size int
		var content []byte
		for idx, val := range b {
			if val == 0 {
				content = b[idx+1:]
				if size, err = strconv.Atoi(string(b[5:idx])); err != nil {
					fmt.Printf("Error converting % x to int at idx: %d", b[5:idx], idx)
				}
				break
			}
		}
		return GitBlobObject{size, content}, nil
	} else {
		fmt.Printf("Content: %s\n", string(b))
	}
	return nil, InvalidObject
}

//解包原生字符串
func (msg *GxMessage) Unpackage() ([]byte, error) {
	if msg.GetLen() == 0 {
		return []byte(""), nil
	}

	if msg.GetLen() == msg.GetUnlen() {
		data := make([]byte, msg.GetLen())
		copy(data[:], msg.Data)
		return data, nil
	} else if msg.GetLen() < msg.GetUnlen() {
		var b bytes.Buffer
		b.Write(msg.Data)
		r, err := zlib.NewReader(&b)
		if err != nil {
			return []byte(""), err
		}
		defer r.Close()

		data := make([]byte, msg.GetUnlen())
		l, _ := r.Read(data)
		if l != int(msg.GetUnlen()) {
			return []byte(""), errors.New("uncompress erro")
		}
		return data, nil
	} else {
		return []byte(""), errors.New("message unpackage erro")
	}
}

func (ms *MapStorage) GetDestination(key string, skipCache bool, transactionID string) (dest *Destination, err error) {
	ms.mu.RLock()
	defer ms.mu.RUnlock()
	cCommit := cacheCommit(transactionID)
	key = utils.DESTINATION_PREFIX + key
	if !skipCache {
		if x, ok := cache.Get(key); ok {
			if x != nil {
				return x.(*Destination), nil
			}
			return nil, utils.ErrNotFound
		}
	}
	if values, ok := ms.dict[key]; ok {
		b := bytes.NewBuffer(values)
		r, err := zlib.NewReader(b)
		if err != nil {
			return nil, err
		}
		out, err := ioutil.ReadAll(r)
		if err != nil {
			return nil, err
		}
		r.Close()
		dest = new(Destination)
		err = ms.ms.Unmarshal(out, dest)
		if err != nil {
			cache.Set(key, dest, cCommit, transactionID)
		}
	} else {
		cache.Set(key, nil, cCommit, transactionID)
		return nil, utils.ErrNotFound
	}
	return
}

func (ms *MapStorage) GetRatingPlan(key string, skipCache bool, transactionID string) (rp *RatingPlan, err error) {
	ms.mu.RLock()
	defer ms.mu.RUnlock()
	key = utils.RATING_PLAN_PREFIX + key
	if !skipCache {
		if x, ok := cache.Get(key); ok {
			if x != nil {
				return x.(*RatingPlan), nil
			}
			return nil, utils.ErrNotFound
		}
	}
	cCommit := cacheCommit(transactionID)
	if values, ok := ms.dict[key]; ok {
		b := bytes.NewBuffer(values)
		r, err := zlib.NewReader(b)
		if err != nil {
			return nil, err
		}
		out, err := ioutil.ReadAll(r)
		if err != nil {
			return nil, err
		}
		r.Close()
		rp = new(RatingPlan)
		err = ms.ms.Unmarshal(out, rp)
	} else {
		cache.Set(key, nil, cCommit, transactionID)
		return nil, utils.ErrNotFound
	}
	cache.Set(key, rp, cCommit, transactionID)
	return
}

// ReadEntity checks the Accept header and reads the content into the entityPointer.
func (r *Request) ReadEntity(entityPointer interface{}) (err error) {
	contentType := r.Request.Header.Get(HEADER_ContentType)
	contentEncoding := r.Request.Header.Get(HEADER_ContentEncoding)

	// check if the request body needs decompression
	if ENCODING_GZIP == contentEncoding {
		gzipReader := currentCompressorProvider.AcquireGzipReader()
		defer currentCompressorProvider.ReleaseGzipReader(gzipReader)
		gzipReader.Reset(r.Request.Body)
		r.Request.Body = gzipReader
	} else if ENCODING_DEFLATE == contentEncoding {
		zlibReader, err := zlib.NewReader(r.Request.Body)
		if err != nil {
			return err
		}
		r.Request.Body = zlibReader
	}

	// lookup the EntityReader, use defaultRequestContentType if needed and provided
	entityReader, ok := entityAccessRegistry.accessorAt(contentType)
	if !ok {
		if len(defaultRequestContentType) != 0 {
			entityReader, ok = entityAccessRegistry.accessorAt(defaultRequestContentType)
		}
		if !ok {
			return NewError(http.StatusBadRequest, "Unable to unmarshal content of type:"+contentType)
		}
	}
	return entityReader.Read(r, entityPointer)
}

func (ms *MapStorage) GetDestination(key string) (dest *Destination, err error) {
	ms.mu.RLock()
	defer ms.mu.RUnlock()
	key = utils.DESTINATION_PREFIX + key
	if values, ok := ms.dict[key]; ok {
		b := bytes.NewBuffer(values)
		r, err := zlib.NewReader(b)
		if err != nil {
			return nil, err
		}
		out, err := ioutil.ReadAll(r)
		if err != nil {
			return nil, err
		}
		r.Close()
		dest = new(Destination)
		err = ms.ms.Unmarshal(out, dest)
		// create optimized structure
		for _, p := range dest.Prefixes {
			CachePush(utils.DESTINATION_PREFIX+p, dest.Id)
		}
	} else {
		return nil, utils.ErrNotFound
	}
	return
}

// Return the bytes for a file.
func (b *Bundle) Bytes(path string) ([]byte, error) {
	file := b.files[path]
	if file == nil {
		return nil, os.ErrNotExist
	}
	if b.compressed {
		if file.uncompressed == nil {
			r, err := zlib.NewReader(bytes.NewReader(file.data))
			if err != nil {
				return nil, err
			}
			wb := &bytes.Buffer{}
			_, err = io.Copy(wb, r)
			if err != nil {
				return nil, err
			}
			if b.retainUncompressed {
				file.uncompressed = wb.Bytes()
			}
			return wb.Bytes(), nil
		} else {
			return file.uncompressed, nil
		}
	}
	return file.data, nil
}

func (ms *MapStorage) GetRatingPlan(key string, skipCache bool) (rp *RatingPlan, err error) {
	key = RATING_PLAN_PREFIX + key
	if !skipCache {
		if x, err := cache2go.GetCached(key); err == nil {
			return x.(*RatingPlan), nil
		} else {
			return nil, err
		}
	}
	if values, ok := ms.dict[key]; ok {
		b := bytes.NewBuffer(values)
		r, err := zlib.NewReader(b)
		if err != nil {
			return nil, err
		}
		out, err := ioutil.ReadAll(r)
		if err != nil {
			return nil, err
		}
		r.Close()
		rp = new(RatingPlan)
		err = ms.ms.Unmarshal(out, rp)
		cache2go.Cache(key, rp)
	} else {
		return nil, errors.New(utils.ERR_NOT_FOUND)
	}
	return
}

// Decrypt decrypts an encrypted message and returns it (plaintext).
// If you have enabled compression, it wil detect it and decompress
// the msg after decrypting it.
func (c SaltSecret) Decrypt(msg []byte) ([]byte, error) {
	if len(msg) < nonceSize+secretbox.Overhead {
		return nil, errors.New("encrypted message length too short")
	}

	nonce := new([nonceSize]byte)
	copy(nonce[:], msg[:nonceSize])

	key, err := scrypt.Key(c.key, nonce[:], 2<<c.NPow, 8, 1, keySize)
	if err != nil {
		return nil, err
	}

	naclKey := new([keySize]byte)
	copy(naclKey[:], key)
	out, ok := secretbox.Open(nil, msg[nonceSize:], nonce, naclKey)
	if !ok {
		return nil, errors.New("could not decrypt message")
	}

	if nonce[23]&compressBit == compressBit {
		r, err := zlib.NewReader(bytes.NewReader(out))
		if err != nil {
			return nil, err
		}
		r.Close()
		out, err = ioutil.ReadAll(r)
		if err != nil {
			return nil, err
		}
	}
	return out, nil
}

func LoadTable(url string) BatchTable {
	var pdf []byte
	if strings.HasPrefix(url, "http") {
		pdf = loadFromUrl(url)
	} else {
		pdf = loadFile(url)
	}
	if pdf == nil {
		log.Printf("PDF file wasn't loaded")
		return nil
	}

	table := make(BatchTable)

	for {
		begin := bytes.Index(pdf, []byte(StreamStartMarker))
		if begin == -1 {
			break
		}
		pdf = pdf[begin+len(StreamStartMarker):]
		end := bytes.Index(pdf, []byte(StreamEndMarker))
		if end == -1 {
			break
		}
		section := pdf[0:end]
		pdf = pdf[end+len(StreamEndMarker):]

		buf := bytes.NewBuffer(section)
		unzipReader, err := zlib.NewReader(buf)
		if err != nil {
			log.Printf("Unzip initialization failed, %v", err)
			continue
		}
		unzipped, err := ioutil.ReadAll(unzipReader)
		if err != nil {
			log.Printf("Unzip failed, %v", err)
			continue
		}
		records := make([]string, 0)
		for _, group := range BTETRE.FindAllSubmatch(unzipped, -1) {
			lines := make([][]byte, 0)
			for _, group := range TextRE.FindAllSubmatch(group[1], -1) {
				lines = append(lines, group[1])
			}
			records = append(records, string(bytes.Join(lines, []byte{})))
		}
		for i := 0; i < len(records)-2; i++ {
			v, err := strconv.ParseInt(records[i], 10, 64)
			if err == nil && v >= 20000000000 && v < 29000000000 {
				id := records[i]
				if _, exists := table[id]; !exists {
					table[id] = make([]BatchUpdate, 0)
				}
				table[id] = append(table[id], BatchUpdate{records[i+1], records[i+2]})
				i += 2
			}
		}
	}
	return table
}

func readBlobData(pos block) ([]byte, error) {
	file, err := os.Open(pos.filename)
	if err != nil {
		return nil, newParserError("file open", err)
	}
	defer file.Close()

	var blob = &osmpbf.Blob{}

	blobData := make([]byte, pos.size)
	file.Seek(pos.offset, 0)
	io.ReadFull(file, blobData)
	err = proto.Unmarshal(blobData, blob)
	if err != nil {
		return nil, newParserError("unmarshaling blob", err)
	}

	// pbf contains (uncompressed) raw or zlibdata
	raw := blob.GetRaw()
	if raw == nil {
		buf := bytes.NewBuffer(blob.GetZlibData())
		r, err := zlib.NewReader(buf)
		if err != nil {
			return nil, newParserError("zlib error", err)
		}
		raw = make([]byte, blob.GetRawSize())
		_, err = io.ReadFull(r, raw)
		if err != nil {
			return nil, newParserError("zlib read error", err)
		}
	}
	return raw, nil
}