Golang camlistore-org-pkg-osutil.GoPackagePath类(方法)实例源码

func doKeyStuff(b *testing.B) keyStuff {
	camliRootPath, err := osutil.GoPackagePath("camlistore.org")
	if err != nil {
		b.Fatal("Package camlistore.org no found in $GOPATH or $GOPATH not defined")
	}
	secretRingFile := filepath.Join(camliRootPath, "pkg", "jsonsign", "testdata", "test-secring.gpg")
	pubKey := `-----BEGIN PGP PUBLIC KEY BLOCK-----

xsBNBEzgoVsBCAC/56aEJ9BNIGV9FVP+WzenTAkg12k86YqlwJVAB/VwdMlyXxvi
bCT1RVRfnYxscs14LLfcMWF3zMucw16mLlJCBSLvbZ0jn4h+/8vK5WuAdjw2YzLs
WtBcjWn3lV6tb4RJz5gtD/o1w8VWxwAnAVIWZntKAWmkcChCRgdUeWso76+plxE5
aRYBJqdT1mctGqNEISd/WYPMgwnWXQsVi3x4z1dYu2tD9uO1dkAff12z1kyZQIBQ
rexKYRRRh9IKAayD4kgS0wdlULjBU98aeEaMz1ckuB46DX3lAYqmmTEL/Rl9cOI0
Enpn/oOOfYFa5h0AFndZd1blMvruXfdAobjVABEBAAE=
=28/7
-----END PGP PUBLIC KEY BLOCK-----`
	return keyStuff{
		secretRingFile: secretRingFile,
		pubKey:         pubKey,
		pubKeyRef:      blob.SHA1FromString(pubKey),
		entityFetcher: &jsonsign.CachingEntityFetcher{
			Fetcher: &jsonsign.FileEntityFetcher{File: secretRingFile},
		},
	}
}

func gitShortlog() *exec.Cmd {
	if !*gitContainer {
		return exec.Command("/bin/bash", "-c", "git log | git shortlog -sen")
	}
	args := []string{"run", "--rm"}
	if inProd {
		args = append(args,
			"-v", "/var/camweb:/var/camweb",
			"--workdir="+prodSrcDir,
		)
	} else {
		hostRoot, err := osutil.GoPackagePath("camlistore.org")
		if err != nil {
			log.Fatal(err)
		}
		log.Printf("Using bind root of %q", hostRoot)
		args = append(args,
			"-v", hostRoot+":"+prodSrcDir,
			"--workdir="+prodSrcDir,
		)
	}
	args = append(args, "camlistore/git", "/bin/bash", "-c", "git log | git shortlog -sen")
	cmd := exec.Command("docker", args...)
	cmd.Stderr = os.Stderr
	return cmd
}

func TestExpansionsInHighlevelConfig(t *testing.T) {
	camroot, err := osutil.GoPackagePath("camlistore.org")
	if err != nil {
		t.Fatalf("failed to find camlistore.org GOPATH root: %v", err)
	}
	const keyID = "26F5ABDA"
	os.Setenv("TMP_EXPANSION_TEST", keyID)
	os.Setenv("TMP_EXPANSION_SECRING", filepath.Join(camroot, filepath.FromSlash("pkg/jsonsign/testdata/test-secring.gpg")))
	// Setting CAMLI_CONFIG_DIR to avoid triggering failInTests in osutil.CamliConfigDir
	defer os.Setenv("CAMLI_CONFIG_DIR", os.Getenv("CAMLI_CONFIG_DIR")) // restore after test
	os.Setenv("CAMLI_CONFIG_DIR", "whatever")
	conf, err := serverinit.Load([]byte(`
{
    "auth": "localhost",
    "listen": ":4430",
    "https": false,
    "identity": ["_env", "${TMP_EXPANSION_TEST}"],
    "identitySecretRing": ["_env", "${TMP_EXPANSION_SECRING}"],
    "googlecloudstorage": ":camlistore-dev-blobs",
    "kvIndexFile": "/tmp/camli-index.kvdb"
}
`))
	if err != nil {
		t.Fatal(err)
	}
	got := fmt.Sprintf("%#v", conf)
	if !strings.Contains(got, keyID) {
		t.Errorf("Expected key %s in resulting low-level config. Got: %s", keyID, got)
	}
}

// setup checks if the camlistore root can be found,
// then sets up closureGitDir and destDir, and returns whether
// we should clone or update in closureGitDir (depending on
// if a .git dir was found).
func setup() string {
	camliRootPath, err := osutil.GoPackagePath("camlistore.org")
	if err != nil {
		log.Fatal("Package camlistore.org not found in $GOPATH (or $GOPATH not defined).")
	}
	destDir = filepath.Join(camliRootPath, "third_party", "closure", "lib")
	closureGitDir = filepath.Join(camliRootPath, "tmp", "closure-lib")
	op := "update"
	_, err = os.Stat(closureGitDir)
	if err != nil {
		if os.IsNotExist(err) {
			err = os.MkdirAll(closureGitDir, 0755)
			if err != nil {
				log.Fatalf("Could not create %v: %v", closureGitDir, err)
			}
			op = "clone"
		} else {
			log.Fatalf("Could not stat %v: %v", closureGitDir, err)
		}
	}
	dotGitPath := filepath.Join(closureGitDir, ".git")
	_, err = os.Stat(dotGitPath)
	if err != nil {
		if os.IsNotExist(err) {
			op = "clone"
		} else {
			log.Fatalf("Could not stat %v: %v", dotGitPath, err)
		}
	}
	return op
}

// camliClosurePage checks if filename is a .js file using closure
// and if yes, if it provides a page in the camlistore namespace.
// It returns that page name, or the empty string otherwise.
func camliClosurePage(filename string) string {
	camliRootPath, err := osutil.GoPackagePath("camlistore.org")
	if err != nil {
		return ""
	}
	fullpath := filepath.Join(camliRootPath, "server", "camlistored", "ui", filename)
	f, err := os.Open(fullpath)
	if err != nil {
		return ""
	}
	defer f.Close()
	br := bufio.NewReader(f)
	for {
		l, err := br.ReadString('\n')
		if err != nil {
			return ""
		}
		if !strings.HasPrefix(l, "goog.") {
			continue
		}
		m := provCamliRx.FindStringSubmatch(l)
		if m != nil {
			return m[2]
		}
	}
	return ""
}

// fileList parses deps.js from the closure repo, as well as the similar
// dependencies generated for the UI js files, and compiles the list of
// js files from the closure lib required for the UI.
func fileList() ([]string, error) {
	camliRootPath, err := osutil.GoPackagePath("camlistore.org")
	if err != nil {
		log.Fatal("Package camlistore.org not found in $GOPATH (or $GOPATH not defined).")
	}
	uiDir := filepath.Join(camliRootPath, "server", "camlistored", "ui")
	closureDepsFile := filepath.Join(closureGitDir, "closure", "goog", "deps.js")

	f, err := os.Open(closureDepsFile)
	if err != nil {
		return nil, err
	}
	defer f.Close()
	allClosureDeps, err := closure.DeepParseDeps(f)
	if err != nil {
		return nil, err
	}

	uiDeps, err := closure.GenDeps(http.Dir(uiDir))
	if err != nil {
		return nil, err
	}
	_, requ, err := closure.ParseDeps(bytes.NewReader(uiDeps))
	if err != nil {
		return nil, err
	}

	nameDone := make(map[string]bool)
	jsfilesDone := make(map[string]bool)
	for _, deps := range requ {
		for _, dep := range deps {
			if _, ok := nameDone[dep]; ok {
				continue
			}
			jsfiles := allClosureDeps[dep]
			for _, filename := range jsfiles {
				if _, ok := jsfilesDone[filename]; ok {
					continue
				}
				jsfilesDone[filename] = true
			}
			nameDone[dep] = true
		}
	}
	jsfiles := []string{
		"AUTHORS",
		"LICENSE",
		"README",
		filepath.Join("closure", "goog", "base.js"),
		filepath.Join("closure", "goog", "css", "common.css"),
		filepath.Join("closure", "goog", "css", "toolbar.css"),
		filepath.Join("closure", "goog", "deps.js"),
	}
	prefix := filepath.Join("closure", "goog")
	for k, _ := range jsfilesDone {
		jsfiles = append(jsfiles, filepath.Join(prefix, k))
	}
	sort.Strings(jsfiles)
	return jsfiles, nil
}

func main() {
	flag.Usage = usage
	flag.Parse()
	checkFlags()

	camDir, err := osutil.GoPackagePath("camlistore.org")
	if err != nil {
		log.Fatalf("Error looking up camlistore.org dir: %v", err)
	}
	dockDir = filepath.Join(camDir, "misc", "docker")

	if *doBuildServer {
		buildDockerImage("go", goDockerImage)
		buildDockerImage("djpeg-static", djpegDockerImage)

		// ctxDir is where we run "docker build" to produce the final
		// "FROM scratch" Docker image.
		ctxDir, err := ioutil.TempDir("", "camli-build")
		if err != nil {
			log.Fatal(err)
		}
		defer os.RemoveAll(ctxDir)

		genCamlistore(ctxDir)
		genDjpeg(ctxDir)
		buildServer(ctxDir)
	}

	if *doUpload {
		uploadDockerImage()
	}
}

func camSrcDir() string {
	if inProd {
		return prodSrcDir
	}
	dir, err := osutil.GoPackagePath("camlistore.org")
	if err != nil {
		log.Fatalf("Failed to find the root of the Camlistore source code via osutil.GoPackagePath: %v", err)
	}
	return dir
}

// TODO(mpl): refactor with twitter
func fakePhoto() string {
	camliDir, err := osutil.GoPackagePath("camlistore.org")
	if err == os.ErrNotExist {
		log.Fatal("Directory \"camlistore.org\" not found under GOPATH/src; are you not running with devcam?")
	}
	if err != nil {
		log.Fatalf("Error searching for \"camlistore.org\" under GOPATH: %v", err)
	}
	return filepath.Join(camliDir, filepath.FromSlash("third_party/glitch/npc_piggy__x1_walk_png_1354829432.png"))
}

// CamliSourceRoot returns the root of the source tree, or an error.
func camliSourceRoot() (string, error) {
	if os.Getenv("GOPATH") == "" {
		return "", errors.New("GOPATH environment variable isn't set; required to run Camlistore integration tests")
	}
	root, err := osutil.GoPackagePath("camlistore.org")
	if err == os.ErrNotExist {
		return "", errors.New("Directory \"camlistore.org\" not found under GOPATH/src; can't run Camlistore integration tests.")
	}
	return root, nil
}

func (c *serverCmd) RunCommand(args []string) error {
	err := c.checkFlags(args)
	if err != nil {
		return cmdmain.UsageError(fmt.Sprint(err))
	}
	c.camliSrcRoot, err = osutil.GoPackagePath("camlistore.org")
	if err != nil {
		return errors.New("Package camlistore.org not found in $GOPATH (or $GOPATH not defined).")
	}
	err = os.Chdir(c.camliSrcRoot)
	if err != nil {
		return fmt.Errorf("Could not chdir to %v: %v", c.camliSrcRoot, err)
	}
	if !c.noBuild {
		for _, name := range []string{"camlistored", "camtool"} {
			err := c.build(name)
			if err != nil {
				return fmt.Errorf("Could not build %v: %v", name, err)
			}
		}
	}
	if err := c.setCamliRoot(); err != nil {
		return fmt.Errorf("Could not setup the camli root: %v", err)
	}
	if err := c.setEnvVars(); err != nil {
		return fmt.Errorf("Could not setup the env vars: %v", err)
	}
	if err := c.setupIndexer(); err != nil {
		return fmt.Errorf("Could not setup the indexer: %v", err)
	}
	if err := c.syncTemplateBlobs(); err != nil {
		return fmt.Errorf("Could not copy the template blobs: %v", err)
	}
	if err := c.setFullClosure(); err != nil {
		return fmt.Errorf("Could not setup the closure lib: %v", err)
	}

	log.Printf("Starting dev server on %v/ui/ with password \"pass%v\"\n",
		os.Getenv("CAMLI_BASEURL"), c.port)

	camliBin := filepath.Join(c.camliSrcRoot, "bin", "camlistored")
	cmdArgs := []string{
		"-configfile=" + filepath.Join(c.camliSrcRoot, "config", "dev-server-config.json"),
		"-listen=" + c.listen}
	cmd := exec.Command(camliBin, cmdArgs...)
	cmd.Stdout = os.Stdout
	cmd.Stderr = os.Stderr
	if err := cmd.Start(); err != nil {
		return fmt.Errorf("Could not start camlistored: %v", err)
	}
	go handleKillCamliSignal(cmd.Process)
	cmd.Wait()
	return nil
}

func main() {
	flag.Usage = usage
	flag.Parse()
	checkFlags()

	camDir, err := osutil.GoPackagePath("camlistore.org")
	if err != nil {
		log.Fatalf("Error looking up camlistore.org dir: %v", err)
	}
	dockDir = filepath.Join(camDir, "misc", "docker")

	if *doImage {
		buildDockerImage("go", goDockerImage)
		buildDockerImage("djpeg-static", djpegDockerImage)
		// ctxDir is where we run "docker build" to produce the final
		// "FROM scratch" Docker image.
		ctxDir, err := ioutil.TempDir("", "camli-build")
		if err != nil {
			log.Fatal(err)
		}
		defer os.RemoveAll(ctxDir)

		genCamlistore(ctxDir)
		genDjpeg(ctxDir)
		buildServer(ctxDir)
	}

	// TODO(mpl): maybe *doBinaries should be done by a separate go program,
	// because the end product is not a docker image. However, we're still
	// using docker all along, and it's convenient for now for code reuse. I
	// can refactor it all out of dock.go afterwards if we like the process.
	if *doBinaries {
		// TODO(mpl): consider using an "official" or trusted existing
		// Go docker image, since we don't do anything special anymore in
		// ours?
		buildDockerImage("go", goDockerImage+"-linux")
		ctxDir, err := ioutil.TempDir("", "camli-build")
		if err != nil {
			log.Fatal(err)
		}
		defer os.RemoveAll(ctxDir)
		genBinaries(ctxDir)
		packBinaries(ctxDir)
	}

	if *doUpload {
		if *doImage {
			uploadDockerImage()
		} else if *doBinaries {
			uploadReleaseTarball()
		}
	}
}

func TestQueryPermanodeLocation(t *testing.T) {
	testQuery(t, func(qt *queryTest) {
		id := qt.id

		p1 := id.NewPlannedPermanode("1")
		p2 := id.NewPlannedPermanode("2")
		p3 := id.NewPlannedPermanode("3")
		id.SetAttribute(p1, "latitude", "51.5")
		id.SetAttribute(p1, "longitude", "0")
		id.SetAttribute(p2, "latitude", "51.5")
		id.SetAttribute(p3, "longitude", "0")

		p4 := id.NewPlannedPermanode("checkin")
		p5 := id.NewPlannedPermanode("venue")
		id.SetAttribute(p4, "camliNodeType", "foursquare.com:checkin")
		id.SetAttribute(p4, "foursquareVenuePermanode", p5.String())
		id.SetAttribute(p5, "latitude", "1.0")
		id.SetAttribute(p5, "longitude", "2.0")

		// Upload a basic image
		camliRootPath, err := osutil.GoPackagePath("camlistore.org")
		if err != nil {
			panic("Package camlistore.org no found in $GOPATH or $GOPATH not defined")
		}
		uploadFile := func(file string, modTime time.Time) blob.Ref {
			fileName := filepath.Join(camliRootPath, "pkg", "search", "testdata", file)
			contents, err := ioutil.ReadFile(fileName)
			if err != nil {
				panic(err)
			}
			br, _ := id.UploadFile(file, string(contents), modTime)
			return br
		}
		fileRef := uploadFile("dude-gps.jpg", time.Time{})

		p6 := id.NewPlannedPermanode("photo")
		id.SetAttribute(p6, "camliContent", fileRef.String())

		sq := &SearchQuery{
			Constraint: &Constraint{
				Permanode: &PermanodeConstraint{
					Location: &LocationConstraint{
						Any: true,
					},
				},
			},
		}
		qt.wantRes(sq, p1, p4, p5, p6)
	})
}

// newTestServer creates a new test server with in memory storage for use in upload tests
func newTestServer(t *testing.T) *httptest.Server {
	camroot, err := osutil.GoPackagePath("camlistore.org")
	if err != nil {
		t.Fatalf("failed to find camlistore.org GOPATH root: %v", err)
	}

	conf := serverconfig.Config{
		Listen:             ":3179",
		HTTPS:              false,
		Auth:               "localhost",
		Identity:           "26F5ABDA",
		IdentitySecretRing: filepath.Join(camroot, filepath.FromSlash("pkg/jsonsign/testdata/test-secring.gpg")),
		MemoryStorage:      true,
		MemoryIndex:        true,
	}

	confData, err := json.MarshalIndent(conf, "", "    ")
	if err != nil {
		t.Fatalf("Could not json encode config: %v", err)
	}

	// Setting CAMLI_CONFIG_DIR to avoid triggering failInTests in osutil.CamliConfigDir
	defer os.Setenv("CAMLI_CONFIG_DIR", os.Getenv("CAMLI_CONFIG_DIR")) // restore after test
	os.Setenv("CAMLI_CONFIG_DIR", "whatever")
	lowConf, err := serverinit.Load(confData)
	if err != nil {
		t.Fatal(err)
	}
	// because these two are normally consumed in camlistored.go
	// TODO(mpl): serverinit.Load should consume these 2 as well. Once
	// consumed, we should keep all the answers as private fields, and then we
	// put accessors on serverinit.Config. Maybe we even stop embedding
	// jsonconfig.Obj in serverinit.Config too, so none of those methods are
	// accessible.
	lowConf.OptionalBool("https", true)
	lowConf.OptionalString("listen", "")

	reindex := false
	var context *http.Request // only used by App Engine. See handlerLoader in serverinit.go
	hi := http.NewServeMux()
	address := "http://" + conf.Listen
	_, err = lowConf.InstallHandlers(hi, address, reindex, context)
	if err != nil {
		t.Fatal(err)
	}

	return httptest.NewServer(hi)
}

func startEmailCommitLoop(errc chan<- error) {
	if *emailsTo == "" {
		return
	}
	if *emailNow != "" {
		dir, err := osutil.GoPackagePath("camlistore.org")
		if err != nil {
			log.Fatal(err)
		}
		if err := emailCommit(dir, *emailNow); err != nil {
			log.Fatal(err)
		}
		os.Exit(0)
	}
	go func() {
		errc <- commitEmailLoop()
	}()
}

func (c *gaeCmd) RunCommand(args []string) error {
	err := c.checkFlags(args)
	if err != nil {
		return cmdmain.UsageError(fmt.Sprint(err))
	}
	c.camliSrcRoot, err = osutil.GoPackagePath("camlistore.org")
	if err != nil {
		return errors.New("Package camlistore.org not found in $GOPATH (or $GOPATH not defined).")
	}
	c.applicationDir = filepath.Join(c.camliSrcRoot, "server", "appengine")
	if _, err := os.Stat(c.applicationDir); err != nil {
		return fmt.Errorf("Appengine application dir not found at %s", c.applicationDir)
	}
	if err = c.checkSDK(); err != nil {
		return err
	}
	if err = c.mirrorSourceRoot(); err != nil {
		return err
	}

	devAppServerBin := filepath.Join(c.sdk, "dev_appserver.py")
	cmdArgs := []string{
		"--skip_sdk_update_check",
		fmt.Sprintf("--port=%s", c.port),
	}
	if c.all {
		cmdArgs = append(cmdArgs, "--host", "0.0.0.0")
	}
	if c.wipe {
		cmdArgs = append(cmdArgs, "--clear_datastore")
	}
	cmdArgs = append(cmdArgs, args...)
	cmdArgs = append(cmdArgs, c.applicationDir)
	cmd := exec.Command(devAppServerBin, cmdArgs...)
	cmd.Stdout = os.Stdout
	cmd.Stderr = os.Stderr
	if err := cmd.Start(); err != nil {
		return fmt.Errorf("Could not start dev_appserver.py: %v", err)
	}
	go handleSignals(cmd.Process)
	cmd.Wait()
	return nil
}

func main() {
	flag.Usage = usage
	flag.Parse()
	checkFlags()

	camDir, err := osutil.GoPackagePath("camlistore.org")
	if err != nil {
		log.Fatalf("Error looking up camlistore.org dir: %v", err)
	}
	dockDir = filepath.Join(camDir, "misc", "docker")

	buildDockerImage("go", goDockerImage)
	// ctxDir is where we run "docker build" to produce the final
	// "FROM scratch" Docker image.
	ctxDir, err := ioutil.TempDir("", "camli-build")
	if err != nil {
		log.Fatal(err)
	}
	defer os.RemoveAll(ctxDir)
	switch {
	case *doImage:
		buildDockerImage("djpeg-static", djpegDockerImage)
		buildDockerImage("zoneinfo", zoneinfoDockerImage)
		genCamlistore(ctxDir)
		genDjpeg(ctxDir)
		genZoneinfo(ctxDir)
		buildServer(ctxDir)
	case *doBinaries:
		genBinaries(ctxDir)
		packBinaries(ctxDir)
	case *doZipSource:
		zipSource(ctxDir)
	}

	if !*doUpload {
		return
	}
	if *doImage {
		uploadDockerImage()
	} else {
		uploadReleaseTarball()
	}
}

func commitEmailLoop() error {
	http.HandleFunc("/mailnow", mailNowHandler)

	go func() {
		for {
			select {
			case tokenc <- true:
			default:
			}
			time.Sleep(15 * time.Second)
		}
	}()

	dir, err := osutil.GoPackagePath("camlistore.org")
	if err != nil {
		return err
	}

	hashes, err := recentCommits(dir)
	if err != nil {
		return err
	}
	for _, commit := range hashes {
		knownCommit[commit] = true
	}
	latestHash.Lock()
	latestHash.s = hashes[0]
	latestHash.Unlock()
	http.HandleFunc("/latesthash", latestHashHandler)

	for {
		pollCommits(dir)

		// Poll every minute or whenever we're forced with the
		// /mailnow handler.
		select {
		case <-time.After(1 * time.Minute):
		case <-fetchc:
			log.Printf("Polling git due to explicit trigger.")
		}
	}
}

// NewWorld returns a new test world.
// It requires that GOPATH is set to find the "camlistore.org" root.
func NewWorld() (*World, error) {
	if os.Getenv("GOPATH") == "" {
		return nil, errors.New("GOPATH environment variable isn't set; required to run Camlistore integration tests")
	}
	root, err := osutil.GoPackagePath("camlistore.org")
	if err == os.ErrNotExist {
		return nil, errors.New("Directory \"camlistore.org\" not found under GOPATH/src; can't run Camlistore integration tests.")
	}
	if err != nil {
		return nil, fmt.Errorf("Error searching for \"camlistore.org\" under GOPATH: %v", err)
	}
	ln, err := net.Listen("tcp", "127.0.0.1:0")
	if err != nil {
		return nil, err
	}

	return &World{
		camRoot:  root,
		listener: ln,
		port:     ln.Addr().(*net.TCPAddr).Port,
	}, nil
}

func main() {
	flag.Usage = usage
	flag.Parse()
	checkFlags()

	var err error
	camDir, err = osutil.GoPackagePath("camlistore.org")
	if err != nil {
		log.Fatalf("Error looking up camlistore.org dir: %v", err)
	}

	if err := genDownloads(); err != nil {
		log.Fatal(err)
	}

	releaseData, err := listDownloads()
	if err != nil {
		log.Fatal(err)
	}

	if *flagStatsFrom != "" && !isWIP() {
		commitStats, err := genCommitStats()
		if err != nil {
			log.Fatal(err)
		}
		releaseData.Stats = commitStats

		notes, err := genReleaseNotes()
		if err != nil {
			log.Fatal(err)
		}
		releaseData.ReleaseNotes = notes
	}

	if err := genMonthlyPage(releaseData); err != nil {
		log.Fatal(err)
	}
}