Golang compress-zlib.NewWriter类(方法)实例源码

func encodeQuery(query []byte) string {
	var compressed_query bytes.Buffer
	w := zlib.NewWriter(&compressed_query)
	w.Write(query)
	w.Close()
	return base64.URLEncoding.EncodeToString(compressed_query.Bytes())
}

func compress(data []byte) []byte {
	var compressedData bytes.Buffer
	writer := zlib.NewWriter(&compressedData)
	writer.Write(data)
	writer.Close()
	return compressedData.Bytes()
}

func (rs *RedisStorage) SetActionPlan(key string, ats *ActionPlan, overwrite bool) (err error) {
	if len(ats.ActionTimings) == 0 {
		// delete the key
		err = rs.db.Cmd("DEL", utils.ACTION_PLAN_PREFIX+key).Err
		cache2go.RemKey(utils.ACTION_PLAN_PREFIX + key)
		return err
	}
	if !overwrite {
		// get existing action plan to merge the account ids
		if existingAts, _ := rs.GetActionPlan(key, true); existingAts != nil {
			if ats.AccountIDs == nil && len(existingAts.AccountIDs) > 0 {
				ats.AccountIDs = make(utils.StringMap)
			}
			for accID := range existingAts.AccountIDs {
				ats.AccountIDs[accID] = true
			}
		}
	}

	result, err := rs.ms.Marshal(ats)
	if err != nil {
		return err
	}
	var b bytes.Buffer
	w := zlib.NewWriter(&b)
	w.Write(result)
	w.Close()
	return rs.db.Cmd("SET", utils.ACTION_PLAN_PREFIX+key, b.Bytes()).Err
}

func main() {
	app := cli.NewApp()
	app.Name = "zlib"
	app.Usage = "A command-line tool for using the zlib compression algorithm."
	app.Action = func(c *cli.Context) {
		var reader io.Reader = os.Stdin
		if c.Bool("decompress") {
			compressorReadCloser, err := zlib.NewReader(reader)
			if err != nil {
				exit(err.Error(), 1)
			}
			if _, err := io.Copy(os.Stdout, compressorReadCloser); err != nil {
				exit(err.Error(), 1)
			}
			compressorReadCloser.Close()
		} else {
			var writer io.Writer = os.Stdout
			compressorWriteCloser := zlib.NewWriter(writer)
			if _, err := io.Copy(compressorWriteCloser, reader); err != nil {
				exit(err.Error(), 1)
			}
			compressorWriteCloser.Close()
		}
	}
	app.Flags = []cli.Flag{
		cli.BoolFlag{
			Name:  "d, decompress",
			Usage: "Decompresses the input instead of compressing the output.",
		},
	}
	app.Run(os.Args)
}

func custom(log, cors, validate bool, f func(w http.ResponseWriter, r *http.Request)) func(w http.ResponseWriter, r *http.Request) {
	return func(w http.ResponseWriter, r *http.Request) {
		addr := r.RemoteAddr
		if ip, found := header(r, "X-Forwarded-For"); found {
			addr = ip
		}
		// compress settings
		ioWriter := w.(io.Writer)
		for _, val := range misc.ParseCsvLine(r.Header.Get("Accept-Encoding")) {
			if val == "gzip" {
				w.Header().Set("Content-Encoding", "gzip")
				g := gzip.NewWriter(w)
				defer g.Close()
				ioWriter = g
				break
			}
			if val == "deflate" {
				w.Header().Set("Content-Encoding", "deflate")
				z := zlib.NewWriter(w)
				defer z.Close()
				ioWriter = z
				break
			}
		}
		writer := &customResponseWriter{Writer: ioWriter, ResponseWriter: w, status: http.StatusOK}

		// route to the controllers
		f(writer, r)

		// access log
		if log && cfg.AccessLog {
			logs.Info.Printf("%s %s %s %s", addr, strconv.Itoa(writer.status), r.Method, r.URL)
		}
	}
}

// serializeChunkData produces the compressed chunk NBT data.
func serializeChunkData(w *nbtChunkWriter) (chunkData []byte, err os.Error) {
	// Reserve room for the chunk data header at the start.
	buffer := bytes.NewBuffer(make([]byte, chunkDataHeaderSize, chunkDataGuessSize))

	if zlibWriter, err := zlib.NewWriter(buffer); err != nil {
		return nil, err
	} else {
		if err = nbt.Write(zlibWriter, w.RootTag()); err != nil {
			zlibWriter.Close()
			return nil, err
		}
		if err = zlibWriter.Close(); err != nil {
			return nil, err
		}
	}
	chunkData = buffer.Bytes()

	// Write chunk data header
	header := chunkDataHeader{
		DataSize: uint32(len(chunkData)) - chunkDataHeaderSize,
		Version:  chunkCompressionZlib,
	}
	buffer = bytes.NewBuffer(chunkData[:0])
	if err = binary.Write(buffer, binary.BigEndian, header); err != nil {
		return nil, err
	}

	return chunkData, nil
}

func custom(log, cors, validate bool, f func(w http.ResponseWriter, r *http.Request)) func(w http.ResponseWriter, r *http.Request) {
	return func(w http.ResponseWriter, r *http.Request) {

		// compress settings
		ioWriter := w.(io.Writer)
		for _, val := range misc.ParseCsvLine(r.Header.Get("Accept-Encoding")) {
			if val == "gzip" {
				w.Header().Set("Content-Encoding", "gzip")
				g := gzip.NewWriter(w)
				defer g.Close()
				ioWriter = g
				break
			}
			if val == "deflate" {
				w.Header().Set("Content-Encoding", "deflate")
				z := zlib.NewWriter(w)
				defer z.Close()
				ioWriter = z
				break
			}
		}
		writer := &customResponseWriter{Writer: ioWriter, ResponseWriter: w, status: 200}

		// route to the controllers
		f(writer, r)
	}
}

func Marshal(compression Compression, out io.Writer, v interface{}) (err error) {
	defer func() {
		if r := recover(); r != nil {
			if s, ok := r.(string); ok {
				err = fmt.Errorf(s)
			} else {
				err = r.(error)
			}
		}
	}()

	if out == nil {
		panic(fmt.Errorf("nbt: Output stream is nil"))
	}

	switch compression {
	case Uncompressed:
		break
	case GZip:
		w := gzip.NewWriter(out)
		defer w.Close()
		out = w
	case ZLib:
		w := zlib.NewWriter(out)
		defer w.Close()
		out = w
	default:
		panic(fmt.Errorf("nbt: Unknown compression type: %d", compression))
	}

	writeRootTag(out, reflect.ValueOf(v))

	return
}

func (f *ZlibFilter) committer(fr FilterRunner, h PluginHelper, wg *sync.WaitGroup) {
	initBatch := make([]byte, 0, 10000)
	f.backChan <- initBatch
	var (
		tag string
		//ok bool
		outBatch []byte
	)
	tag = f.ZlibTag

	for outBatch = range f.batchChan {
		pack, e := h.PipelinePack(f.msgLoopCount)
		if e != nil {
			fr.LogError(e)
			break
		}
		var b bytes.Buffer
		w := zlib.NewWriter(&b)
		w.Write(outBatch)
		w.Close()

		tagField, _ := message.NewField("ZlibTag", tag, "")
		pack.Message.AddField(tagField)
		pack.Message.SetUuid(uuid.NewRandom())
		pack.Message.SetPayload(b.String())
		fr.Inject(pack)

		outBatch = outBatch[:0]
		f.backChan <- outBatch
	}
	wg.Done()
}

func (ms *MongoStorage) SetActionPlan(key string, ats *ActionPlan, overwrite bool) error {
	// clean dots from account ids map
	if len(ats.ActionTimings) == 0 {
		cache2go.RemKey(utils.ACTION_PLAN_PREFIX + key)
		err := ms.db.C(colApl).Remove(bson.M{"key": key})
		if err != mgo.ErrNotFound {
			return err
		}
		return nil
	}
	if !overwrite {
		// get existing action plan to merge the account ids
		if existingAts, _ := ms.GetActionPlan(key, true); existingAts != nil {
			if ats.AccountIDs == nil && len(existingAts.AccountIDs) > 0 {
				ats.AccountIDs = make(utils.StringMap)
			}
			for accID := range existingAts.AccountIDs {
				ats.AccountIDs[accID] = true
			}
		}
	}
	result, err := ms.ms.Marshal(ats)
	if err != nil {
		return err
	}
	var b bytes.Buffer
	w := zlib.NewWriter(&b)
	w.Write(result)
	w.Close()
	_, err = ms.db.C(colApl).Upsert(bson.M{"key": key}, &struct {
		Key   string
		Value []byte
	}{Key: key, Value: b.Bytes()})
	return err
}

func Compress(data []byte) bytes.Buffer {
	var b bytes.Buffer
	w := zlib.NewWriter(&b)
	w.Write(data)
	w.Close()
	return b
}

func (f *file) Close() error {
	if !f.closed {
		f.f.Lock()
		defer f.f.Unlock()
		if !f.closed {
			if f.f.Mode&ModeCompress != 0 {
				var buf bytes.Buffer
				zw := zlib.NewWriter(&buf)
				if _, err := zw.Write(f.data); err != nil {
					return err
				}
				if err := zw.Close(); err != nil {
					return err
				}
				if buf.Len() < len(f.data) {
					f.f.Data = buf.Bytes()
				} else {
					f.f.Mode &= ^ModeCompress
					f.f.Data = f.data
				}
			} else {
				f.f.Data = f.data
			}
			f.closed = true
		}
	}
	return nil
}

func compress(data []byte) []byte {
	var b bytes.Buffer
	w := zlib.NewWriter(&b)
	w.Write(data)
	w.Close()
	return b.Bytes()
}

func (rs *RedisStorage) SetActionPlan(key string, ats *ActionPlan, overwrite bool, transactionID string) (err error) {
	cCommit := cacheCommit(transactionID)
	if len(ats.ActionTimings) == 0 {
		// delete the key
		err = rs.Cmd("DEL", utils.ACTION_PLAN_PREFIX+key).Err
		cache.RemKey(utils.ACTION_PLAN_PREFIX+key, cCommit, transactionID)
		return err
	}
	if !overwrite {
		// get existing action plan to merge the account ids
		if existingAts, _ := rs.GetActionPlan(key, true, transactionID); existingAts != nil {
			if ats.AccountIDs == nil && len(existingAts.AccountIDs) > 0 {
				ats.AccountIDs = make(utils.StringMap)
			}
			for accID := range existingAts.AccountIDs {
				ats.AccountIDs[accID] = true
			}
		}
		// do not keep this in cache (will be obsolete)
		cache.RemKey(utils.ACTION_PLAN_PREFIX+key, cCommit, transactionID)
	}
	result, err := rs.ms.Marshal(ats)
	if err != nil {
		return err
	}
	var b bytes.Buffer
	w := zlib.NewWriter(&b)
	w.Write(result)
	w.Close()
	err = rs.Cmd("SET", utils.ACTION_PLAN_PREFIX+key, b.Bytes()).Err
	cache.RemKey(utils.ACTION_PLAN_PREFIX+key, cCommit, transactionID)
	return
}

func save(r redis.AsyncClient, key string, obj interface{}, w http.ResponseWriter) {
	var b bytes.Buffer

	z := zlib.NewWriter(&b)
	defer z.Close()

	je := json.NewEncoder(z)

	err := je.Encode(obj)
	if err != nil {
		log.Fatal("Failed to json Encode with error: ", err)
	}
	z.Flush()

	f, rerr := r.Set(key, b.Bytes())
	if rerr != nil {
		panic(rerr)
	}
	_, rerr, timeout := f.TryGet(50000000000)
	if rerr != nil {
		panic(rerr)
	}
	if timeout {
		savetimeout++
		log.Println("save timeout! count: ", savetimeout)
		fmt.Fprintf(w, "Save failed for %s", key)
	}
}

func encode(txt *mdnsTxt) ([]string, error) {
	b, err := json.Marshal(txt)
	if err != nil {
		return nil, err
	}

	var buf bytes.Buffer
	defer buf.Reset()

	w := zlib.NewWriter(&buf)
	if _, err := w.Write(b); err != nil {
		return nil, err
	}
	w.Close()

	encoded := hex.EncodeToString(buf.Bytes())

	// individual txt limit
	if len(encoded) <= 255 {
		return []string{encoded}, nil
	}

	// split encoded string
	var record []string

	for len(encoded) > 255 {
		record = append(record, encoded[:255])
		encoded = encoded[255:]
	}

	record = append(record, encoded)

	return record, nil
}

/**
Attempts to encode the response according to the client's Accept-Encoding
header. If there is an error, or if the encoding requests aren't supported
then the original content is returned.

Encoding type:
 * deflate (zlib stream)
 * gzip

This should be the last module loaded
*/
func EncodeResponse(ctx *Context, content interface{}) (interface{}, error) {
	var compressed bytes.Buffer
	var output io.WriteCloser

	if len(ctx.Request.Header["Accept-Encoding"]) > 0 {
		for _, opt := range ctx.Request.Header["Accept-Encoding"] {
			if strings.Index(opt, "gzip") >= 0 {
				output = gzip.NewWriter(&compressed)
				ctx.SetHeader("Content-Encoding", "gzip", true)
			} else if strings.Index(opt, "deflate") >= 0 {
				output = zlib.NewWriter(&compressed)
				ctx.SetHeader("Content-Encoding", "deflate", true)
			}
		}
	}

	if output != nil {
		_, err := output.Write(content.([]byte))
		if err != nil {
			ctx.Server.Logger.Printf("EncodeResponse write failed: %s", err)
			return content, &WebError{500, err.Error()}
		}
		err = output.Close()
		return compressed.Bytes(), nil
	}

	return content, nil
}

func (i *Index) save(idxFile string) error {
	if idxFile == "" {
		err := fmt.Errorf("Yikes! Cannot save index to disk because no file was specified.")
		return err
	}
	fp, err := ioutil.TempFile(path.Dir(idxFile), "idx-build")
	if err != nil {
		return err
	}
	zfp := zlib.NewWriter(fp)
	i.m.RLock()
	defer i.m.RUnlock()
	enc := gob.NewEncoder(zfp)
	err = enc.Encode(i)
	zfp.Close()
	if err != nil {
		fp.Close()
		return err
	}
	err = fp.Close()
	if err != nil {
		return nil
	}
	return os.Rename(fp.Name(), idxFile)
}

//打包原生字符串
func (msg *GxMessage) Package(buf []byte) error {
	l := len(buf)
	if l == 0 {
		return nil
	}

	var b bytes.Buffer
	c := false

	//小于指定长度不用检查是否需要压缩
	if l > 10 {
		w := zlib.NewWriter(&b)
		w.Write(buf)
		w.Close()
		c = true
	}

	//压缩后长度比原来小，就保存压缩数据
	if c && b.Len() < l {
		msg.SetUnlen(uint16(l))
		msg.SetLen(uint16(b.Len()))
		msg.Data = make([]byte, b.Len())
		copy(msg.Data[:], b.Bytes())
	} else {
		msg.SetUnlen(uint16(l))
		msg.SetLen(uint16(l))
		msg.Data = make([]byte, l)
		copy(msg.Data[:], buf)
	}
	return nil
}

func gitFlattenObject(sha1 string) (io.Reader, error) {
	kind, err := gitCatKind(sha1)
	if err != nil {
		return nil, errgo.Notef(err, "flatten: kind(%s) failed", sha1)
	}
	size, err := gitCatSize(sha1)
	if err != nil {
		return nil, errgo.Notef(err, "flatten: size(%s) failed", sha1)
	}
	r, err := gitCatData(sha1, kind)
	if err != nil {
		return nil, errgo.Notef(err, "flatten: data(%s) failed", sha1)
	}
	// move to exp/git
	pr, pw := io.Pipe()
	go func() {
		zw := zlib.NewWriter(pw)
		if _, err := fmt.Fprintf(zw, "%s %d\x00", kind, size); err != nil {
			pw.CloseWithError(errgo.Notef(err, "writing git format header failed"))
			return
		}
		if _, err := io.Copy(zw, r); err != nil {
			pw.CloseWithError(errgo.Notef(err, "copying git data failed"))
			return
		}
		if err := zw.Close(); err != nil {
			pw.CloseWithError(errgo.Notef(err, "zlib close failed"))
			return
		}
		pw.Close()
	}()
	return pr, nil
}