作者：jquinte
项目：gogres
func QueryKeys(c appengine.Context, cursor string) ([]Key, string, error) {
var items []Key
q := datastore.NewQuery("Key")
if cursor != "" {
dCursor, err := datastore.DecodeCursor(string(cursor))
if err == nil {
q = q.Start(dCursor)
}
}
t := q.Run(c)
for i := 0; i < 10; i++ {
var item Key
k, err := t.Next(&item)
if err == datastore.Done {
break
}
if err != nil {
return items, "", nil
}
item.Id = k.StringID()
item.Agent.Get(c, item.AgentId)
item.Portal.Get(c, item.PortalId)
items = append(items, item)
}
returnedCursor, err := t.Cursor()
if err != nil {
return items, "", err
}
return items, returnedCursor.String(), nil
}
作者：koffeinsourc
项目：notreddi
// GetNewestItems returns the latest number elements for a specific namespace
func GetNewestItems(c appengine.Context, namespace string, limit int, cursor string) ([]Item, string, error) {
q := datastore.NewQuery("Item").
Filter("Namespace =", namespace).
Order("-CreatedAt").
Limit(limit)
if cursor, err := datastore.DecodeCursor(cursor); err == nil {
q = q.Start(cursor)
}
var is []Item
var err error
t := q.Run(c)
for {
var i Item
_, err = t.Next(&i)
if err == datastore.Done {
break
}
is = append(is, i)
if err != nil {
c.Errorf("Error fetching next item for namespace %v: %v", namespace, err)
return nil, "", err
}
}
if cursor, err := t.Cursor(); err == nil {
return is, cursor.String(), nil
}
return nil, "", err
}
作者：knights
项目：goslide
func query2(w http.ResponseWriter, r *http.Request) {
c := appengine.NewContext(r)
// start cursor1 OMIT
q := datastore.NewQuery("Book").Filter("Author=", "author-1").Order("-CreatedAt")
pCursor := r.FormValue("cursor")
if pCursor != "" {
cursor, err := datastore.DecodeCursor(pCursor)
if err != nil {
http.Error(w, err.Error(), http.StatusInternalServerError)
return
}
q.Start(cursor)
}
// end cursor1 OMIT
// start cursor2 OMIT
var books []Book
t := q.Run(c)
for i := 0; i < 10; i++ {
var book Book
key, err := t.Next(&book)
if err == datastore.Done {
break
}
if err != nil {
http.Error(w, err.Error(), http.StatusInternalServerError)
return
}
c.Debugf("#v", key)
books = append(books, book)
}
// end cursor2 OMIT
response := struct {
Cursor string
Books []Book
}{
Books: books,
}
// start cursor3 OMIT
response.Books = books
if cursor, err := t.Cursor(); err == nil {
response.Cursor = cursor.String()
}
// end cursor3 OMIT
je := json.NewEncoder(w)
if err := je.Encode(response); err != nil {
http.Error(w, err.Error(), http.StatusInternalServerError)
return
}
}
作者：Pretlis
项目：gorea
func GetFeed(c mpg.Context, w http.ResponseWriter, r *http.Request) {
gn := goon.FromContext(c)
f := Feed{Url: r.FormValue("f")}
var stars []string
wg := sync.WaitGroup{}
fk := gn.Key(&f)
q := datastore.NewQuery(gn.Kind(&Story{})).Ancestor(fk).KeysOnly()
q = q.Order("-" + IDX_COL)
if cur := r.FormValue("c"); cur != "" {
if dc, err := datastore.DecodeCursor(cur); err == nil {
q = q.Start(dc)
}
} else {
// grab the stars list on the first run
wg.Add(1)
go c.Step("stars", func(c mpg.Context) {
gn := goon.FromContext(c)
usk := starKey(c, f.Url, "")
q := datastore.NewQuery(gn.Kind(&UserStar{})).Ancestor(gn.Key(usk).Parent()).KeysOnly()
keys, _ := gn.GetAll(q, nil)
stars = make([]string, len(keys))
for i, key := range keys {
stars[i] = starID(key)
}
wg.Done()
})
}
iter := gn.Run(q)
var stories []*Story
for i := 0; i < 20; i++ {
if k, err := iter.Next(nil); err == nil {
stories = append(stories, &Story{
Id: k.StringID(),
Parent: k.Parent(),
})
} else if err == datastore.Done {
break
} else {
serveError(w, err)
return
}
}
cursor := ""
if ic, err := iter.Cursor(); err == nil {
cursor = ic.String()
}
gn.GetMulti(&stories)
wg.Wait()
b, _ := json.Marshal(struct {
Cursor string
Stories []*Story
Stars []string `json:",omitempty"`
}{
Cursor: cursor,
Stories: stories,
Stars: stars,
})
w.Write(b)
}
作者：prodigen
项目：gorea
func GetStars(c mpg.Context, w http.ResponseWriter, r *http.Request) {
gn := goon.FromContext(c)
cu := user.Current(c)
u := User{Id: cu.ID}
q := datastore.NewQuery(gn.Key(&UserStar{}).Kind()).
Ancestor(gn.Key(&u)).
Order("-c").
Limit(20)
if cur := r.FormValue("c"); cur != "" {
if dc, err := datastore.DecodeCursor(cur); err == nil {
q = q.Start(dc)
}
}
iter := gn.Run(q)
stars := make(map[string]int64)
var us UserStar
var stories []*Story
for {
if k, err := iter.Next(&us); err == nil {
stars[starID(k)] = us.Created.Unix()
stories = append(stories, &Story{
Id: k.StringID(),
Parent: gn.Key(&Feed{Url: k.Parent().StringID()}),
})
} else if err == datastore.Done {
break
} else {
serveError(w, err)
return
}
}
cursor := ""
if ic, err := iter.Cursor(); err == nil {
cursor = ic.String()
}
var smap map[string][]*Story
if len(stories) > 0 {
gn.GetMulti(&stories)
smap = make(map[string][]*Story)
for _, s := range stories {
f := s.Parent.StringID()
smap[f] = append(smap[f], s)
}
}
b, _ := json.Marshal(struct {
Cursor string
Stories map[string][]*Story
Stars map[string]int64
}{
Cursor: cursor,
Stories: smap,
Stars: stars,
})
w.Write(b)
}
作者：timburk
项目：openrada
func applyCursor(query_in *datastore.Query, cursor_in string) (query_out *datastore.Query, err error) {
if cursor_in != "" {
cursor, err := datastore.DecodeCursor(cursor_in)
if err == nil {
return query_in.Start(cursor), nil
} else {
return query_in, err
}
} else {
return query_in, nil
}
}
作者：ZhiqinYan
项目：msgpac
func decodeDatastoreCursorValue(d *Decoder, v reflect.Value) error {
s, err := d.DecodeString()
if err != nil {
return err
}
cursor, err := ds.DecodeCursor(s)
if err != nil {
return err
}
v.Set(reflect.ValueOf(cursor))
return nil
}
作者：rahna
项目：gorea
func SitemapFeed(c mpg.Context, w http.ResponseWriter, r *http.Request) {
vars := mux.Vars(r)
feed := vars["feed"]
fk, err := datastore.DecodeKey(feed)
if err != nil {
serveError(w, err)
return
}
bf := base64.URLEncoding.EncodeToString([]byte(fk.StringID()))
q := datastore.NewQuery("S").KeysOnly().Ancestor(fk)
q = q.Limit(Limit)
cs := r.FormValue("c")
if len(cs) > 0 {
if cur, err := datastore.DecodeCursor(cs); err == nil {
q = q.Start(cur)
}
}
stories := make(map[string]string)
it := q.Run(c)
for {
k, err := it.Next(nil)
if err == datastore.Done {
break
} else if err != nil {
c.Errorf("next error: %v", err)
break
}
stories[k.StringID()] = base64.URLEncoding.EncodeToString([]byte(k.StringID()))
}
cs = ""
if len(stories) == Limit {
if cur, err := it.Cursor(); err == nil {
cs = cur.String()
}
}
if err := templates.ExecuteTemplate(w, "sitemap-feed.html", struct {
Feed, Feed64 string
Stories map[string]string
Cursor string
}{
Feed: feed,
Feed64: bf,
Stories: stories,
Cursor: cs,
}); err != nil {
c.Errorf("%v", err)
serveError(w, err)
return
}
}
作者：bashtia
项目：gorea
func BackendStart(c mpg.Context, w http.ResponseWriter, r *http.Request) {
return
const sz = 100
ic := 0
var f func(appengine.Context)
var cs string
f = func(c appengine.Context) {
gn := goon.FromContext(c)
c.Errorf("ic: %d", ic)
wg := sync.WaitGroup{}
wg.Add(sz)
var j int64
q := datastore.NewQuery("F").KeysOnly()
if cs != "" {
if cur, err := datastore.DecodeCursor(cs); err == nil {
q = q.Start(cur)
c.Errorf("cur start: %v", cur)
}
}
it := q.Run(c)
for j = 0; j < sz; j++ {
k, err := it.Next(nil)
c.Errorf("%v: %v, %v", j, k, err)
if err != nil {
c.Criticalf("err: %v", err)
return
}
go func(k *datastore.Key) {
f := Feed{Url: k.StringID()}
if err := gn.Get(&f); err == nil {
f.Subscribe(c)
}
wg.Done()
}(k)
}
cur, err := it.Cursor()
if err == nil {
cs = cur.String()
}
wg.Wait()
ic++
runtime.RunInBackground(c, f)
}
runtime.RunInBackground(c, f)
}
作者：knights
项目：sandbo
// 画像のメタデータ一覧をDSから取得します。
// TODO: 表示する画像数を絞る必要がないなら、Cursor必要ないかも。
func GetImages(c appengine.Context, cursorStr string) ([]Image, string, error) {
q := datastore.NewQuery("Image").Order("-Date")
if len(cursorStr) != 0 {
cursor, err := datastore.DecodeCursor(cursorStr)
if err != nil {
return []Image{}, "", err
}
q = q.Start(cursor)
}
images := []Image{}
iter := q.Run(c)
isNext := true
for {
var img Image
_, err := iter.Next(&img)
if err == datastore.Done {
isNext = false
break
}
if err != nil {
c.Errorf("fetching next Person: %s", err.Error())
break
}
err = img.setThumbnailURL(thumbnailsLongestSide, false)
if err != nil {
c.Errorf("%s", err.Error())
break
}
images = append(images, img)
}
if isNext {
next_cursor, err := iter.Cursor()
if err != nil {
c.Errorf("%s", err.Error())
return []Image{}, "", err
}
return images, next_cursor.String(), nil
} else {
return images, "", nil
}
}
作者：jquinte
项目：gogres
func GetPortals(c appengine.Context, labels string, checkfavorited bool, favorited []string, cursor string) ([]Portal, string, error) {
q := datastore.NewQuery("Portal")
if cursor != "" {
dCursor, err := datastore.DecodeCursor(string(cursor))
if err == nil {
q = q.Start(dCursor)
}
}
if len(labels) == 0 {
q = q.Limit(30)
} else {
splits := strings.Split(labels, " ")
c.Infof("query....%s", splits)
q = q.Filter("Labels=", splits[0]).Limit(10)
}
var portals []Portal
t := q.Run(c)
for i := 0; i < 10; i++ {
var portal Portal
_, err := t.Next(&portal)
if err == datastore.Done {
break
}
if err != nil {
return portals, "", err
}
if _, err := datastore.NewQuery("Key").Filter("PortalId=", portal.Id).GetAll(c, &portal.Keys); err != nil {
return portals, "", err
}
if checkfavorited {
for _, portalid := range favorited {
if portalid == portal.Id {
portals = append(portals, portal)
}
}
} else {
portals = append(portals, portal)
}
}
cursor1, err := t.Cursor()
if err != nil {
return portals, "", err
}
return portals, cursor1.String(), nil
}
作者：nightexcessiv
项目：netrac
func doExpire(c appengine.Context, expireTime time.Time, cursorString string) {
query := getExpiredQuery(expireTime).KeysOnly()
if len(cursorString) > 0 {
if cursor, err := datastore.DecodeCursor(cursorString); err != nil {
c.Errorf("Failed to decode cursor: %s", err)
return
} else {
query = query.Start(cursor)
}
}
for {
toDelete := make([]*datastore.Key, 0, 100)
for queryIterator := query.Run(c); ; {
peerKey, err := queryIterator.Next(nil)
if len(toDelete) >= 100 {
break
} else if err == datastore.Done {
c.Infof("Done finding expired peers")
break
} else if err != nil {
c.Criticalf("Failed to get next peer: %#v (%s)", err, err)
return
}
toDelete = append(toDelete, peerKey)
}
deleted := len(toDelete)
c.Infof("Deleting %d expired peers", deleted)
if err := ds.DeleteMulti(c, toDelete); err != nil {
c.Criticalf("Failed to delete peers: %#v (%s)", err, err)
return
}
if deleted <= 0 {
break
}
}
c.Infof("Finished deleting expired peers")
}
作者：huluw
项目：gorea
func DeleteOldFeeds(c mpg.Context, w http.ResponseWriter, r *http.Request) {
ctx := appengine.Timeout(c, time.Minute)
gn := goon.FromContext(c)
q := datastore.NewQuery(gn.Kind(&Feed{})).Filter("n=", timeMax).KeysOnly()
if cur, err := datastore.DecodeCursor(r.FormValue("c")); err == nil {
q = q.Start(cur)
}
it := q.Run(ctx)
done := false
var tasks []*taskqueue.Task
for i := 0; i < 10000 && len(tasks) < 100; i++ {
k, err := it.Next(nil)
if err == datastore.Done {
c.Criticalf("done")
done = true
break
} else if err != nil {
c.Errorf("err: %v", err)
continue
}
values := make(url.Values)
values.Add("f", k.StringID())
tasks = append(tasks, taskqueue.NewPOSTTask("/tasks/delete-old-feed", values))
}
if len(tasks) > 0 {
c.Errorf("deleting %v feeds", len(tasks))
if _, err := taskqueue.AddMulti(c, tasks, ""); err != nil {
c.Errorf("err: %v", err)
}
}
if !done {
if cur, err := it.Cursor(); err == nil {
values := make(url.Values)
values.Add("c", cur.String())
taskqueue.Add(c, taskqueue.NewPOSTTask("/tasks/delete-old-feeds", values), "")
} else {
c.Errorf("err: %v", err)
}
}
}
作者：jgastei
项目：irakurrig
func GetFeed(c mpg.Context, w http.ResponseWriter, r *http.Request) {
gn := goon.FromContext(c)
f := Feed{Url: r.FormValue("f")}
fk := gn.Key(&f)
q := datastore.NewQuery(gn.Key(&Story{}).Kind()).Ancestor(fk).KeysOnly()
q = q.Order("-" + IDX_COL)
if c := r.FormValue("c"); c != "" {
if dc, err := datastore.DecodeCursor(c); err == nil {
q = q.Start(dc)
}
}
iter := gn.Run(q)
var stories []*Story
for i := 0; i < 20; i++ {
if k, err := iter.Next(nil); err == nil {
stories = append(stories, &Story{
Id: k.StringID(),
Parent: k.Parent(),
})
} else if err == datastore.Done {
break
} else {
serveError(w, err)
return
}
}
cursor := ""
if ic, err := iter.Cursor(); err == nil {
cursor = ic.String()
}
gn.GetMulti(&stories)
b, _ := json.Marshal(struct {
Cursor string
Stories []*Story
}{
Cursor: cursor,
Stories: stories,
})
w.Write(b)
}
作者：rahna
项目：gorea
func Sitemap(c mpg.Context, w http.ResponseWriter, r *http.Request) {
q := datastore.NewQuery("F").KeysOnly()
q = q.Limit(Limit)
cs := r.FormValue("c")
if len(cs) > 0 {
if cur, err := datastore.DecodeCursor(cs); err == nil {
q = q.Start(cur)
}
}
var keys []*datastore.Key
it := q.Run(c)
for {
k, err := it.Next(nil)
if err == datastore.Done {
break
} else if err != nil {
c.Errorf("next error: %v", err)
break
}
keys = append(keys, k)
}
cs = ""
if len(keys) == Limit {
if cur, err := it.Cursor(); err == nil {
cs = cur.String()
}
}
if err := templates.ExecuteTemplate(w, "sitemap.html", struct {
Keys []*datastore.Key
Cursor string
}{
Keys: keys,
Cursor: cs,
}); err != nil {
c.Errorf("%v", err)
serveError(w, err)
return
}
}
作者：e4
项目：cayle
func (it *Iterator) Next() bool {
if it.offset+1 < len(it.buffer) {
it.offset++
it.result = &Token{Kind: it.kind, Hash: it.buffer[it.offset]}
return true
}
if it.done {
return false
}
// Reset buffer and offset
it.offset = 0
it.buffer = make([]string, 0, bufferSize)
// Create query
// TODO (panamafrancis) Keys only query?
q := datastore.NewQuery(it.kind).Limit(bufferSize)
if !it.isAll {
// Filter on the direction {subject,objekt...}
q = q.Filter(it.dir.String()+" =", it.name)
}
// Get last cursor position
cursor, err := datastore.DecodeCursor(it.last)
if err == nil {
q = q.Start(cursor)
}
// Buffer the keys of the next 50 matches
t := q.Run(it.qs.context)
for {
// Quirk of the datastore, you cannot pass a nil value to to Next()
// even if you just want the keys
var k *datastore.Key
skip := false
if it.kind == quadKind {
temp := new(QuadEntry)
k, err = t.Next(temp)
// Skip if quad has been deleted
if len(temp.Added) <= len(temp.Deleted) {
skip = true
}
} else {
temp := new(NodeEntry)
k, err = t.Next(temp)
// Skip if node has been deleted
if temp.Size == 0 {
skip = true
}
}
if err == datastore.Done {
it.done = true
break
}
if err != nil {
glog.Errorf("Error fetching next entry %v", err)
it.err = err
return false
}
if !skip {
it.buffer = append(it.buffer, k.StringID())
}
}
// Save cursor position
cursor, err = t.Cursor()
if err == nil {
it.last = cursor.String()
}
// Protect against bad queries
if it.done && len(it.buffer) == 0 {
glog.Warningf("Query did not return any results")
return false
}
// First result
it.result = &Token{Kind: it.kind, Hash: it.buffer[it.offset]}
return true
}
作者：rrudduc
项目：golang-stuf
func UpdateFeeds(c mpg.Context, w http.ResponseWriter, r *http.Request) {
q := datastore.NewQuery("F").KeysOnly().Filter("n <=", time.Now())
q = q.Limit(100)
cs := r.FormValue("c")
hasCursor := false
if len(cs) > 0 {
if cur, err := datastore.DecodeCursor(cs); err == nil {
q = q.Start(cur)
hasCursor = true
c.Infof("starting at %v", cur)
} else {
c.Errorf("cursor error %v", err.Error())
}
}
if !hasCursor {
qs, err := taskqueue.QueueStats(c, []string{"update-feed"}, 0)
if err != nil || !qs[0].OldestETA.IsZero() {
c.Errorf("already %v (%v) tasks", qs[0].Tasks, qs[0].Executed1Minute)
return
}
}
var keys []*datastore.Key
it := q.Run(Timeout(c, time.Second*60))
for {
k, err := it.Next(nil)
if err == datastore.Done {
break
} else if err != nil {
c.Errorf("next error: %v", err.Error())
break
}
keys = append(keys, k)
}
if len(keys) == 0 {
c.Errorf("no results")
return
} else {
cur, err := it.Cursor()
if err != nil {
c.Errorf("to cur error %v", err.Error())
} else {
c.Infof("add with cur %v", cur)
t := taskqueue.NewPOSTTask(routeUrl("update-feeds"), url.Values{
"c": {cur.String()},
})
taskqueue.Add(c, t, "update-feed")
}
}
c.Infof("updating %d feeds", len(keys))
var tasks []*taskqueue.Task
for _, k := range keys {
tasks = append(tasks, taskqueue.NewPOSTTask(routeUrl("update-feed"), url.Values{
"feed": {k.StringID()},
}))
}
var ts []*taskqueue.Task
const taskLimit = 100
for len(tasks) > 0 {
if len(tasks) > taskLimit {
ts = tasks[:taskLimit]
tasks = tasks[taskLimit:]
} else {
ts = tasks
tasks = tasks[0:0]
}
if _, err := taskqueue.AddMulti(c, ts, "update-feed"); err != nil {
c.Errorf("taskqueue error: %v", err.Error())
}
}
}
作者：couchbaselab
项目：statstor
func batchMap(c appengine.Context, w http.ResponseWriter, r *http.Request) {
start := time.Now()
if !queueMore(c) {
c.Infof("Too many jobs queued, backing off")
http.Error(w, "Busy", 503)
return
}
q := datastore.NewQuery(r.FormValue("type")).KeysOnly()
if cstr := r.FormValue("cursor"); cstr != "" {
cursor, err := datastore.DecodeCursor(cstr)
maybePanic(err)
c.Infof("Starting from cursor %v", cstr)
q = q.Start(cursor)
}
keys := []string{}
finished := false
t := q.Run(c)
for i := 0; i < 10000; i++ {
k, err := t.Next(nil)
if err == datastore.Done {
finished = true
break
} else if err != nil {
http.Error(w, err.Error(), 500)
return
}
keys = append(keys, k.Encode())
}
c.Infof("Got %v %v keys in %v, finished=%v",
len(keys), r.FormValue("type"), time.Since(start), finished)
for len(keys) > 0 && r.FormValue("next") != "" {
subkeys := keys
if len(subkeys) > 100 {
subkeys = keys[:100]
keys = keys[100:]
} else {
keys = nil
}
buf := &bytes.Buffer{}
z := gzip.NewWriter(buf)
e := json.NewEncoder(z)
maybePanic(e.Encode(subkeys))
maybePanic(z.Flush())
maybePanic(z.Close())
c.Infof("Queueing %v with %v bytes of compressed keys",
mapStage2, buf.Len())
task := &taskqueue.Task{
Path: r.FormValue("next"),
Payload: buf.Bytes(),
}
_, err := taskqueue.Add(c, task, mapStage2)
maybePanic(err)
}
if !finished {
cursor, err := t.Cursor()
maybePanic(err)
c.Infof("Requesting more from %v", cursor.String())
r.Form.Set("cursor", cursor.String())
taskqueue.Add(c, taskqueue.NewPOSTTask("/batch/map", r.Form), mapStage1)
}
w.WriteHeader(201)
}
作者：drborge
项目：app
func (runner *runner) StartFrom(cursor string) *runner {
c, _ := datastore.DecodeCursor(cursor)
runner.query = runner.query.Start(c)
return runner
}
作者：icz
项目：iczagp
//.........这里部分代码省略.........
case fv("submitNextPage") != "":
page++
}
pageSize := p.Account.GetLogsPageSize()
if ps := fv("pageSize"); ps != "" && ps != strconv.Itoa(pageSize) {
// Page size has been changed (on Settings page), drop cursors, render page 1
page = 1
cursorsString = ""
cursors = make([]string, 0, 1)
}
// 'ts all good, proceed with the query:
q := datastore.NewQuery(ds.ENameGPS).Filter(ds.PNameDevKeyID+"=", devID)
if !before.IsZero() {
q = q.Filter(ds.PNameCreated+"<", before)
}
if !after.IsZero() {
q = q.Filter(ds.PNameCreated+">", after)
}
if areaCode >= 0 {
q = q.Filter(ds.PNameAreaCodes+"=", areaCode)
}
q = q.Order("-" + ds.PNameCreated).Limit(pageSize)
var records = make([]*ds.GPS, 0, pageSize)
// If there is a cursor, set it.
// Page - cursor index mapping: cursors[page-2]
// 1st page: no cursor, 2nd page: cursors[0], 3nd page: cursors[1], ...
if page > 1 && page <= len(cursors)+1 {
var cursor datastore.Cursor
if cursor, p.Err = datastore.DecodeCursor(cursors[page-2]); p.Err != nil {
return
}
q = q.Start(cursor)
}
// Iterate over the results:
t := q.Run(c)
for {
r := new(ds.GPS)
_, err := t.Next(r)
if err == datastore.Done {
break
}
if err != nil {
// Datastore error
p.Err = err
return
}
records = append(records, r)
r.Dd = -1 // For now, will be set if applicable
if r.Track() {
// Check the previous (in time) record and calculate distance.
// If previous is not a Track, check the one before that etc.
for i := len(records) - 2; i >= 0; i-- {
if prev := records[i]; prev.Track() {
prev.Dd = Distance(r.GeoPoint.Lat, r.GeoPoint.Lng, prev.GeoPoint.Lat, prev.GeoPoint.Lng)
prev.Dt = prev.Created.Sub(r.Created)
break
}
}
}
}
