Golang camlistore-org-pkg-schema.WriteFileFromReader类(方法)实例源码

func populatePacked2(t *testing.T, s *storage) (wants []storagetest.StreamerTestOpt) {
	const fileSize = 1 << 20
	data := randBytes(fileSize)
	_, err := schema.WriteFileFromReader(s, "first-half.dat", bytes.NewReader(data[:fileSize/2]))
	if err != nil {
		t.Fatalf("WriteFileFromReader: %v", err)
	}
	_, err = schema.WriteFileFromReader(s, "second-half.dat", bytes.NewReader(data[fileSize/2:]))
	if err != nil {
		t.Fatalf("WriteFileFromReader: %v", err)
	}
	return nil
}

// urlFileRef slurps urlstr from the net, writes to a file and returns its
// fileref or "" on error
func (r *run) urlFileRef(urlstr, filename string) string {
	im := r.im
	im.mu.Lock()
	if br, ok := im.imageFileRef[urlstr]; ok {
		im.mu.Unlock()
		return br.String()
	}
	im.mu.Unlock()

	res, err := r.HTTPClient().Get(urlstr)
	if err != nil {
		log.Printf("couldn't get image: %v", err)
		return ""
	}
	defer res.Body.Close()

	fileRef, err := schema.WriteFileFromReader(r.Host.Target(), filename, res.Body)
	if err != nil {
		r.errorf("couldn't write file: %v", err)
		return ""
	}

	im.mu.Lock()
	defer im.mu.Unlock()
	im.imageFileRef[urlstr] = fileRef
	return fileRef.String()
}

func TestHandlerRightRef(t *testing.T) {
	b := test.Blob{Contents: "Foo"}
	storage := new(test.Fetcher)
	ref, err := schema.WriteFileFromReader(storage, "", b.Reader())
	if err != nil {
		t.Fatal(err)
	}
	if err != nil {
		t.Fatal(err)
	}

	ts := httptest.NewServer(createVideothumbnailHandler(ref, storage))
	defer ts.Close()

	resp, err := http.Get(ts.URL + "/" + ref.String())

	if err != nil {
		t.Fatal(err)
	}
	if resp.StatusCode != 200 {
		t.Fatalf("expected 200 status: %v", resp)
	}
	content, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		t.Fatal(err)
	}
	if string(content) != b.Contents {
		t.Errorf("excepted handler to serve data")
	}
}

// urlFileRef slurps urlstr from the net, writes to a file and returns its
// fileref or "" on error
func (r *run) urlFileRef(urlstr string) string {
	if urlstr == "" {
		return ""
	}
	im := r.im
	im.mu.Lock()
	if br, ok := im.urlFileRef[urlstr]; ok {
		im.mu.Unlock()
		return br.String()
	}
	im.mu.Unlock()

	res, err := r.HTTPClient().Get(urlstr)
	if err != nil {
		log.Printf("couldn't get file: %v", err)
		return ""
	}
	defer res.Body.Close()

	filename := urlstr[strings.LastIndex(urlstr, "/")+1:]
	fileRef, err := schema.WriteFileFromReader(r.Host.Target(), filename, res.Body)
	if err != nil {
		log.Printf("couldn't write file: %v", err)
		return ""
	}

	im.mu.Lock()
	defer im.mu.Unlock()
	im.urlFileRef[urlstr] = fileRef
	return fileRef.String()
}

func (h *mutFileHandle) Release(req *fuse.ReleaseRequest, intr fuse.Intr) fuse.Error {
	if h.tmp == nil {
		log.Printf("Release called on camli mutFileHandle without a tempfile set")
		return fuse.EIO
	}
	log.Printf("mutFileHandle release.")
	_, err := h.tmp.Seek(0, 0)
	if err != nil {
		log.Println("mutFileHandle.Release:", err)
		return fuse.EIO
	}
	var n int64
	br, err := schema.WriteFileFromReader(h.f.fs.client, h.f.name, readerutil.CountingReader{Reader: h.tmp, N: &n})
	if err != nil {
		log.Println("mutFileHandle.Release:", err)
		return fuse.EIO
	}
	h.f.setContent(br, n)

	h.tmp.Close()
	os.Remove(h.tmp.Name())
	h.tmp = nil

	return nil
}

// urlFileRef slurps urlstr from the net, writes to a file and returns its
// fileref or "" on error or if urlstr was empty.
func (r *run) urlFileRef(urlstr, filename string) string {
	im := r.im
	im.mu.Lock()
	if br, ok := im.imageFileRef[urlstr]; ok {
		im.mu.Unlock()
		return br.String()
	}
	im.mu.Unlock()

	if urlstr == "" {
		return ""
	}
	res, err := ctxutil.Client(r.Context()).Get(urlstr)
	if err != nil {
		log.Printf("foursquare: couldn't fetch image %q: %v", urlstr, err)
		return ""
	}
	defer res.Body.Close()

	fileRef, err := schema.WriteFileFromReader(r.Host.Target(), filename, res.Body)
	if err != nil {
		r.errorf("couldn't write file: %v", err)
		return ""
	}

	im.mu.Lock()
	defer im.mu.Unlock()
	im.imageFileRef[urlstr] = fileRef
	return fileRef.String()
}

func (ih *ImageHandler) writeToCache(tr io.Reader, name string) (blob.Ref, error) {
	br, err := schema.WriteFileFromReader(ih.Cache, name, tr)
	if err != nil {
		return br, errors.New("failed to cache " + name + ": " + err.Error())
	}
	if imageDebug {
		log.Printf("Image Cache: saved as %v\n", br)
	}
	return br, nil
}

func populatePacked(t *testing.T, s *storage) (wants []storagetest.StreamerTestOpt) {
	const fileSize = 5 << 20
	const fileName = "foo.dat"
	fileContents := randBytes(fileSize)
	_, err := schema.WriteFileFromReader(s, fileName, bytes.NewReader(fileContents))
	if err != nil {
		t.Fatalf("WriteFileFromReader: %v", err)
	}
	return nil
}

func TestPackTwoIdenticalfiles(t *testing.T) {
	const fileSize = 1 << 20
	fileContents := randBytes(fileSize)
	testPack(t,
		func(sto blobserver.Storage) (err error) {
			if _, err = schema.WriteFileFromReader(sto, "a.txt", bytes.NewReader(fileContents)); err != nil {
				return
			}
			if _, err = schema.WriteFileFromReader(sto, "b.txt", bytes.NewReader(fileContents)); err != nil {
				return
			}
			return
		},
		func(pt *packTest) { pt.sto.packGate = syncutil.NewGate(1) }, // one pack at a time
		wantNumLargeBlobs(1),
		wantNumSmallBlobs(1), // just the "b.txt" file schema blob
		okayWithoutMeta("sha1-cb4399f6b3b31ace417e1ec9326f9818bb3f8387"),
	)
}

func storageAndBlobRef(t *testing.T) (blobserver.Storage, blob.Ref) {
	storage := new(test.Fetcher)
	inFile, err := os.Open(testFilepath)
	if err != nil {
		t.Fatal(err)
	}
	ref, err := schema.WriteFileFromReader(storage, "small.webm", inFile)
	if err != nil {
		t.Fatal(err)
	}
	return storage, ref
}

func TestPackNoDelete(t *testing.T) {
	const fileSize = 1 << 20
	const fileName = "foo.dat"
	fileContents := randBytes(fileSize)
	testPack(t,
		func(sto blobserver.Storage) error {
			_, err := schema.WriteFileFromReader(sto, fileName, bytes.NewReader(fileContents))
			return err
		},
		func(pt *packTest) { pt.sto.skipDelete = true },
		wantNumLargeBlobs(1),
		wantNumSmallBlobs(15), // empirically
	)
}

func (ui *UIHandler) serveUploadHelper(rw http.ResponseWriter, req *http.Request) {
	ret := make(map[string]interface{})
	defer httputil.ReturnJSON(rw, ret)

	if ui.root.Storage == nil {
		ret["error"] = "No BlobRoot configured"
		ret["errorType"] = "server"
		return
	}

	mr, err := req.MultipartReader()
	if err != nil {
		ret["error"] = "reading body: " + err.Error()
		ret["errorType"] = "server"
		return
	}

	got := make([]map[string]interface{}, 0)
	for {
		part, err := mr.NextPart()
		if err == io.EOF {
			break
		}
		if err != nil {
			ret["error"] = "reading body: " + err.Error()
			ret["errorType"] = "server"
			break
		}
		fileName := part.FileName()
		if fileName == "" {
			continue
		}
		br, err := schema.WriteFileFromReader(ui.root.Storage, fileName, part)

		if err == nil {
			got = append(got, map[string]interface{}{
				"filename": part.FileName(),
				"formname": part.FormName(),
				"fileref":  br.String(),
			})
		} else {
			ret["error"] = "writing to blobserver: " + err.Error()
			return
		}
	}
	ret["got"] = got
}

func writeToCache(cache blobserver.Storage, thumbBytes []byte, name string) (br blob.Ref, err error) {
	tr := bytes.NewReader(thumbBytes)
	if len(thumbBytes) < constants.MaxBlobSize {
		br = blob.SHA1FromBytes(thumbBytes)
		_, err = blobserver.Receive(cache, br, tr)
	} else {
		// TODO: don't use rolling checksums when writing this. Tell
		// the filewriter to use 16 MB chunks instead.
		br, err = schema.WriteFileFromReader(cache, name, tr)
	}
	if err != nil {
		return br, errors.New("failed to cache " + name + ": " + err.Error())
	}
	if imageDebug {
		log.Printf("Image Cache: saved as %v\n", br)
	}
	return br, nil
}

func TestPackLarge(t *testing.T) {
	if testing.Short() {
		t.Skip("skipping in short mode")
	}
	const fileSize = 17 << 20 // more than 16 MB, so more than one zip
	const fileName = "foo.dat"
	fileContents := randBytes(fileSize)

	hash := blob.NewHash()
	hash.Write(fileContents)
	wholeRef := blob.RefFromHash(hash)

	pt := testPack(t,
		func(sto blobserver.Storage) error {
			_, err := schema.WriteFileFromReader(sto, fileName, bytes.NewReader(fileContents))
			return err
		},
		wantNumLargeBlobs(2),
		wantNumSmallBlobs(0),
	)

	// Verify we wrote the correct "w:*" meta rows.
	got := map[string]string{}
	want := map[string]string{
		"w:" + wholeRef.String():        "17825792 2",
		"w:" + wholeRef.String() + ":0": "sha1-9b4a3d114c059988075c87293c86ee7cbc6f4af5 37 0 16709479",
		"w:" + wholeRef.String() + ":1": "sha1-fe6326ac6b389ffe302623e4a501bfc8c6272e8e 37 16709479 1116313",
	}
	if err := sorted.Foreach(pt.sto.meta, func(key, value string) error {
		if strings.HasPrefix(key, "b:") {
			return nil
		}
		got[key] = value
		return nil
	}); err != nil {
		t.Fatal(err)
	}
	if !reflect.DeepEqual(got, want) {
		t.Errorf("'w:*' meta rows = %v; want %v", got, want)
	}

	// And verify we can read it back out.
	pt.testOpenWholeRef(t, wholeRef, fileSize)
}

func (im *imp) Run(intr importer.Interrupt) (err error) {
	log.Printf("Running dummy importer.")
	defer func() {
		log.Printf("Dummy importer returned: %v", err)
	}()
	root, err := im.host.RootObject()
	if err != nil {
		return err
	}
	fileRef, err := schema.WriteFileFromReader(im.host.Target(), "foo.txt", strings.NewReader("Some file.\n"))
	if err != nil {
		return err
	}
	obj, err := root.ChildPathObject("foo.txt")
	if err != nil {
		return err
	}
	return obj.SetAttr("camliContent", fileRef.String())
}

// urlFileRef slurps urlstr from the net, writes to a file and returns its
// fileref or "" on error
func (im *imp) urlFileRef(urlstr string) string {
	if br, ok := im.imageFileRef[urlstr]; ok {
		return br.String()
	}
	res, err := im.host.HTTPClient().Get(urlstr)
	if err != nil {
		log.Printf("couldn't get image: %v", err)
		return ""
	}
	defer res.Body.Close()

	fileRef, err := schema.WriteFileFromReader(im.host.Target(), "category.png", res.Body)
	if err != nil {
		log.Printf("couldn't write file: %v", err)
		return ""
	}

	im.imageFileRef[urlstr] = fileRef
	return fileRef.String()
}

func TestPackNormal(t *testing.T) {
	const fileSize = 5 << 20
	const fileName = "foo.dat"
	fileContents := randBytes(fileSize)

	hash := blob.NewHash()
	hash.Write(fileContents)
	wholeRef := blob.RefFromHash(hash)

	pt := testPack(t,
		func(sto blobserver.Storage) error {
			_, err := schema.WriteFileFromReader(sto, fileName, bytes.NewReader(fileContents))
			return err
		},
		wantNumLargeBlobs(1),
		wantNumSmallBlobs(0),
	)
	// And verify we can read it back out.
	pt.testOpenWholeRef(t, wholeRef, fileSize)
}

// TODO(aa):
// * Parallelize: http://golang.org/doc/effective_go.html#concurrency
// * Do more than one "page" worth of results
// * Report progress and errors back through host interface
// * All the rest of the metadata (see photoMeta)
// * Conflicts: For all metadata changes, prefer any non-imported claims
// * Test!
func (im *imp) importPhoto(parent *importer.Object, photo *photosSearchItem) error {
	filename := fmt.Sprintf("%s.%s", photo.Id, photo.Originalformat)
	photoNode, err := parent.ChildPathObject(filename)
	if err != nil {
		return err
	}

	// Import all the metadata. SetAttrs() is a no-op if the value hasn't changed, so there's no cost to doing these on every run.
	// And this way if we add more things to import, they will get picked up.
	if err := photoNode.SetAttrs(
		"flickrId", photo.Id,
		"title", photo.Title,
		"description", photo.Description.Content); err != nil {
		return err
	}

	// Import the photo itself. Since it is expensive to fetch the image, we store its lastupdate and only refetch if it might have changed.
	if photoNode.Attr("flickrLastupdate") == photo.Lastupdate {
		return nil
	}
	res, err := im.flickrRequest(photo.URL, url.Values{})
	if err != nil {
		log.Printf("Flickr importer: Could not fetch %s: %s", photo.URL, err)
		return err
	}
	defer res.Body.Close()

	fileRef, err := schema.WriteFileFromReader(im.host.Target(), filename, res.Body)
	if err != nil {
		return err
	}
	if err := photoNode.SetAttr("camliContent", fileRef.String()); err != nil {
		return err
	}
	// Write lastupdate last, so that if any of the preceding fails, we will try again next time.
	if err := photoNode.SetAttr("flickrLastupdate", photo.Lastupdate); err != nil {
		return err
	}

	return nil
}

func storePhoto(p photo) (string, error) {
	srcFile := localPathOf(p)

	f, err := os.Open(srcFile)
	if err != nil {
		return "", err
	}
	defer f.Close()

	fileRef, err := schema.WriteFileFromReader(camliClient, p.Id+"."+p.Extension, f)

	res, err := camliClient.UploadNewPermanode()
	if err != nil {
		return "", err
	}
	perma := res.BlobRef

	p.Description = cleanHTML(p.Description)

	claims := []*schema.Builder{}
	claims = append(claims, schema.NewSetAttributeClaim(perma, "camliContent", fileRef.String()))
	claims = append(claims, schema.NewSetAttributeClaim(perma, "title", mkTitle(p.Description)))
	claims = append(claims, schema.NewSetAttributeClaim(perma, "description", p.Description))
	for _, t := range p.Tags {
		claims = append(claims, schema.NewAddAttributeClaim(perma, "tag", t))
	}
	if p.Cat == "Public" {
		claims = append(claims, schema.NewSetAttributeClaim(perma, "camliAccess", "public"))
	}

	grp := syncutil.Group{}
	for _, claimBuilder := range claims {
		claim := claimBuilder.Blob()
		grp.Go(func() error {
			_, err := camliClient.UploadAndSignBlob(claim)
			return err
		})
	}

	return perma.String(), grp.Err()
}

func (ui *UIHandler) serveUploadHelper(rw http.ResponseWriter, req *http.Request) {
	if ui.root.Storage == nil {
		httputil.ServeJSONError(rw, httputil.ServerError("No BlobRoot configured"))
		return
	}

	mr, err := req.MultipartReader()
	if err != nil {
		httputil.ServeJSONError(rw, httputil.ServerError("reading body: "+err.Error()))
		return
	}

	var got []*uploadHelperGotItem
	for {
		part, err := mr.NextPart()
		if err == io.EOF {
			break
		}
		if err != nil {
			httputil.ServeJSONError(rw, httputil.ServerError("reading body: "+err.Error()))
			break
		}
		fileName := part.FileName()
		if fileName == "" {
			continue
		}
		br, err := schema.WriteFileFromReader(ui.root.Storage, fileName, part)
		if err != nil {
			httputil.ServeJSONError(rw, httputil.ServerError("writing to blobserver: "+err.Error()))
			return
		}
		got = append(got, &uploadHelperGotItem{
			FileName: part.FileName(),
			FormName: part.FormName(),
			FileRef:  br,
		})
	}

	httputil.ReturnJSON(rw, &uploadHelperResponse{Got: got})
}