作者:reedobrie
项目:gorea
func UpdateFeeds(c mpg.Context, w http.ResponseWriter, r *http.Request) {
q := datastore.NewQuery("F").KeysOnly().Filter("n <=", time.Now())
q = q.Limit(10 * 60 * 2) // 10/s queue, 2 min cron
it := q.Run(appengine.Timeout(c, time.Minute))
tc := make(chan *taskqueue.Task)
done := make(chan bool)
i := 0
u := routeUrl("update-feed")
go taskSender(c, "update-feed", tc, done)
for {
k, err := it.Next(nil)
if err == datastore.Done {
break
} else if err != nil {
c.Errorf("next error: %v", err.Error())
break
}
tc <- taskqueue.NewPOSTTask(u, url.Values{
"feed": {k.StringID()},
})
i++
}
close(tc)
<-done
c.Infof("updating %d feeds", i)
}
作者:hdonna
项目:gorea
func UpdateFeed(c mpg.Context, w http.ResponseWriter, r *http.Request) {
gn := goon.FromContext(appengine.Timeout(c, time.Minute))
url := r.FormValue("feed")
if url == "" {
c.Errorf("empty update feed")
return
}
c.Debugf("update feed %s", url)
last := len(r.FormValue("last")) > 0
f := Feed{Url: url}
s := ""
defer func() {
gn.Put(&Log{
Parent: gn.Key(&f),
Id: time.Now().UnixNano(),
Text: "UpdateFeed - " + s,
})
}()
if err := gn.Get(&f); err == datastore.ErrNoSuchEntity {
c.Errorf("no such entity - " + url)
s += "NSE"
return
} else if err != nil {
s += "err - " + err.Error()
return
} else if last {
// noop
}
if time.Now().Before(f.NextUpdate) {
c.Errorf("feed %v already updated: %v", url, f.NextUpdate)
s += "already updated"
return
}
feedError := func(err error) {
s += "feed err - " + err.Error()
f.Errors++
v := f.Errors + 1
const max = 24 * 7
if v > max {
v = max
} else if f.Errors == 1 {
v = 0
}
f.NextUpdate = time.Now().Add(time.Hour * time.Duration(v))
gn.Put(&f)
c.Warningf("error with %v (%v), bump next update to %v, %v", url, f.Errors, f.NextUpdate, err)
}
if feed, stories, err := fetchFeed(c, f.Url, f.Url); err == nil {
if err := updateFeed(c, f.Url, feed, stories, false, false, last); err != nil {
feedError(err)
} else {
s += "success"
}
} else {
feedError(err)
}
f.Subscribe(c)
}
作者:hugo
项目:complaint
func reportHandler(w http.ResponseWriter, r *http.Request) {
if r.FormValue("date") == "" {
var params = map[string]interface{}{
"Yesterday": date.NowInPdt().AddDate(0, 0, -1),
}
if err := templates.ExecuteTemplate(w, "report-form", params); err != nil {
http.Error(w, err.Error(), http.StatusInternalServerError)
}
return
}
c := appengine.Timeout(appengine.NewContext(r), 60*time.Second) // Default has a 5s timeout
s, e, _ := widget.FormValueDateRange(r)
opt := ReportOptions{
ClassB_OnePerFlight: widget.FormValueCheckbox(r, "classb_oneperflight"),
ClassB_LocalDataOnly: widget.FormValueCheckbox(r, "classb_localdataonly"),
Skimmer_AltitudeTolerance: widget.FormValueFloat64(w, r, "skimmer_altitude_tolerance"),
Skimmer_MinDurationNM: widget.FormValueFloat64(w, r, "skimmer_min_duration_nm"),
}
if fix := strings.ToUpper(r.FormValue("waypoint")); fix != "" {
if _, exists := sfo.KFixes[fix]; !exists {
http.Error(w, fmt.Sprintf("Waypoint '%s' not known", fix), http.StatusInternalServerError)
return
}
opt.Waypoint = fix
}
reportWriter(c, w, r, s, e, opt, r.FormValue("reportname"), r.FormValue("resultformat"))
}
作者:hugo
项目:complaint
// Upgrade the set of complaints for each user.
func upgradeUserHandler(w http.ResponseWriter, r *http.Request) {
c := appengine.Timeout(appengine.NewContext(r), 30*time.Second)
cdb := ComplaintDB{C: c, Memcache: false}
p := types.ComplainerProfile{}
if err := p.Base64Decode(r.FormValue("profile")); err != nil {
c.Errorf("upgradeUserHandler: decode failed: %v", err)
http.Error(w, err.Error(), http.StatusInternalServerError)
return
}
// Get *all* the complaints for this person, unfiltered.
var data = []types.Complaint{}
q := datastore.
NewQuery(kComplaintKind).
Ancestor(cdb.emailToRootKey(p.EmailAddress)).
Order("Timestamp")
keys, err := q.GetAll(c, &data)
if err != nil {
c.Errorf("upgradeUserHandler/%s: GetAll failed: %v", p.EmailAddress, err)
http.Error(w, err.Error(), http.StatusInternalServerError)
return
}
nDeleted, nUpdated := 0, 0
str := ""
for i, complaint := range data {
FixupComplaint(&complaint, keys[i]) // Put the key where cdb.* expect to find it
deleteMe := i < len(data)-1 && ComplaintsAreEquivalent(data[i], data[i+1])
noProfile := complaint.Profile.EmailAddress == ""
//str += fmt.Sprintf(" [%03d] [DEL=%5v,PRO=%5v] %s\n", i, deleteMe, noProfile, complaint)
if true {
if deleteMe {
if err := cdb.DeleteComplaints([]string{complaint.DatastoreKey}, p.EmailAddress); err != nil {
c.Errorf("upgradeUserHandler/%s: deletecomplaints failed: %v", p.EmailAddress, err)
http.Error(w, err.Error(), http.StatusInternalServerError)
return
}
nDeleted++
} else if noProfile {
complaint.Profile = p
if err := cdb.UpdateComplaint(complaint, p.EmailAddress); err != nil {
c.Errorf("upgradeUserHandler/%s: updatecomplaints failed: %v", p.EmailAddress, err)
http.Error(w, err.Error(), http.StatusInternalServerError)
return
}
nUpdated++
}
}
}
str += fmt.Sprintf(" *** upgraded {%s} [tot=%d, del=%d, upd=%d]\n",
p.EmailAddress, len(data), nDeleted, nUpdated)
c.Infof(" -- Upgrade for %s --\n%s", p.EmailAddress, str)
w.Write([]byte(fmt.Sprintf("OK, upgraded %s\n", p.EmailAddress)))
}
作者:hugo
项目:complaint
func downloadHandler(w http.ResponseWriter, r *http.Request) {
session := sessions.Get(r)
if session.Values["email"] == nil {
http.Error(w, "session was empty; no cookie ? is this browser in privacy mode ?",
http.StatusInternalServerError)
return
}
c := appengine.Timeout(appengine.NewContext(r), 60*time.Second)
cdb := complaintdb.ComplaintDB{C: c}
cap, err := cdb.GetAllByEmailAddress(session.Values["email"].(string), true)
if err != nil {
http.Error(w, err.Error(), http.StatusInternalServerError)
return
}
_ = cap
filename := date.NowInPdt().Format("complaints-20060102.csv")
w.Header().Set("Content-Type", "application/csv")
w.Header().Set("Content-Disposition", fmt.Sprintf("attachment; filename=\"%s\"", filename))
cols := []string{
"Date", "Time(PDT)", "Notes", "Speedbrakes", "Loudness", "Activity",
"Flightnumber", "Origin", "Destination", "Speed(Knots)", "Altitude(Feet)",
"Lat", "Long", "Registration", "Callsign",
"VerticalSpeed(FeetPerMin)", "Dist2(km)", "Dist3(km)",
}
csvWriter := csv.NewWriter(w)
csvWriter.Write(cols)
for _, c := range cap.Complaints {
a := c.AircraftOverhead
speedbrakes := ""
if c.HeardSpeedbreaks {
speedbrakes = "y"
}
r := []string{
c.Timestamp.Format("2006/01/02"),
c.Timestamp.Format("15:04:05"),
c.Description, speedbrakes, fmt.Sprintf("%d", c.Loudness), c.Activity,
a.FlightNumber, a.Origin, a.Destination,
fmt.Sprintf("%.0f", a.Speed), fmt.Sprintf("%.0f", a.Altitude),
fmt.Sprintf("%.5f", a.Lat), fmt.Sprintf("%.5f", a.Long),
a.Registration, a.Callsign, fmt.Sprintf("%.0f", a.VerticalSpeed),
fmt.Sprintf("%.1f", c.Dist2KM), fmt.Sprintf("%.1f", c.Dist3KM),
}
if err := csvWriter.Write(r); err != nil {
http.Error(w, err.Error(), http.StatusInternalServerError)
}
}
csvWriter.Flush()
}
作者:speedlan
项目:wc
func NewDriver(ctx appengine.Context, kind string, logger wcg.Logger) *Driver {
return &Driver{
ctx: appengine.Timeout(ctx, DefaultTimeout),
kind: kind,
Logger: logger,
EnableOperationLogging: DataStoreConfig.EnableOperationLogging,
Timeout: DefaultTimeout,
namespace: "",
PutSplitThreshold: DefaultPutSplitTreshold,
}
}
作者:hugo
项目:complaint
func cannedSerfr1ComplaintsHandler(w http.ResponseWriter, r *http.Request) {
s, e := date.WindowForYesterday()
e = e.Add(-1 * time.Second)
format := "list"
if r.FormValue("csv") != "" {
format = "csv"
}
c := appengine.Timeout(appengine.NewContext(r), 60*time.Second) // Default has a 5s timeout
reportWriter(c, w, r, s, e, ReportOptions{}, "serfr1complaints", format)
}
作者:hugo
项目:complaint
func cannedAdsbClassBHandler(w http.ResponseWriter, r *http.Request) {
s, e := date.WindowForYesterday()
e = e.Add(-1 * time.Second)
opt := ReportOptions{}
format := "list"
if r.FormValue("csv") != "" {
format = "csv"
}
c := appengine.Timeout(appengine.NewContext(r), 60*time.Second) // Default has a 5s timeout
reportWriter(c, w, r, s, e, opt, "adsbclassb", format)
}
作者:vishnuv
项目:davin
func CronFetchHandler(w http.ResponseWriter, r *http.Request) {
c := appengine.Timeout(appengine.NewContext(r), 30*time.Minute)
q := datastore.NewQuery("Queue").KeysOnly()
keys, _ := q.GetAll(c, nil)
db := DB{c}
start := time.Now()
for _, v := range keys {
db.FetchUser(strconv.FormatInt(v.IntID(), 10))
}
finish := time.Since(start)
c.Infof("Finished cron fetch, took %s", finish)
fmt.Fprint(w, "fetching users")
}
作者:sringe
项目:gorea
func UpdateFeeds(c mpg.Context, w http.ResponseWriter, r *http.Request) {
q := datastore.NewQuery("F").KeysOnly().Filter("n <=", time.Now())
q = q.Limit(10 * 60 * 20) // 10/s queue, 20 min cron
var keys []*datastore.Key
it := q.Run(appengine.Timeout(c, time.Second*60))
for {
k, err := it.Next(nil)
if err == datastore.Done {
break
} else if err != nil {
c.Errorf("next error: %v", err.Error())
break
}
keys = append(keys, k)
}
if len(keys) == 0 {
c.Errorf("no results")
return
}
c.Infof("updating %d feeds", len(keys))
var tasks []*taskqueue.Task
for _, k := range keys {
tasks = append(tasks, taskqueue.NewPOSTTask(routeUrl("update-feed"), url.Values{
"feed": {k.StringID()},
}))
}
var ts []*taskqueue.Task
const taskLimit = 100
for len(tasks) > 0 {
if len(tasks) > taskLimit {
ts = tasks[:taskLimit]
tasks = tasks[taskLimit:]
} else {
ts = tasks
tasks = tasks[0:0]
}
if _, err := taskqueue.AddMulti(c, ts, "update-feed"); err != nil {
c.Errorf("taskqueue error: %v", err.Error())
}
}
}
作者:MiCHiL
项目:testing-gae-taskqueue-quot
func runTestTQ(
c appengine.Context,
r *http.Request,
) {
//u.Debugf(c, "payload: %q", len(payload))
c = appengine.Timeout(c, deadline)
_, err := taskqueueAdd(c, &taskqueue.Task{
Payload: payload,
Method: "PULL",
}, "pull")
if err != nil {
u.Errorf(c, "%v", err)
return
}
delayTestTQ.Call(c, "")
return
}
作者:biggtfis
项目:gorea
func (g *Goon) putMemcache(srcs []interface{}, exists []byte) error {
items := make([]*memcache.Item, len(srcs))
payloadSize := 0
for i, src := range srcs {
toSerialize := src
if exists[i] == 0 {
toSerialize = nil
}
data, err := serializeStruct(toSerialize)
if err != nil {
g.error(err)
return err
}
key, _, err := g.getStructKey(src)
if err != nil {
return err
}
// payloadSize will overflow if we push 2+ gigs on a 32bit machine
payloadSize += len(data)
items[i] = &memcache.Item{
Key: memkey(key),
Value: data,
}
}
memcacheTimeout := MemcachePutTimeoutSmall
if payloadSize >= MemcachePutTimeoutThreshold {
memcacheTimeout = MemcachePutTimeoutLarge
}
errc := make(chan error)
go func() {
errc <- memcache.SetMulti(appengine.Timeout(g.Context, memcacheTimeout), items)
}()
g.putMemoryMulti(srcs, exists)
err := <-errc
if appengine.IsTimeoutError(err) {
g.timeoutError(err)
err = nil
} else if err != nil {
g.error(err)
}
return err
}
作者:huluw
项目:gorea
func DeleteOldFeeds(c mpg.Context, w http.ResponseWriter, r *http.Request) {
ctx := appengine.Timeout(c, time.Minute)
gn := goon.FromContext(c)
q := datastore.NewQuery(gn.Kind(&Feed{})).Filter("n=", timeMax).KeysOnly()
if cur, err := datastore.DecodeCursor(r.FormValue("c")); err == nil {
q = q.Start(cur)
}
it := q.Run(ctx)
done := false
var tasks []*taskqueue.Task
for i := 0; i < 10000 && len(tasks) < 100; i++ {
k, err := it.Next(nil)
if err == datastore.Done {
c.Criticalf("done")
done = true
break
} else if err != nil {
c.Errorf("err: %v", err)
continue
}
values := make(url.Values)
values.Add("f", k.StringID())
tasks = append(tasks, taskqueue.NewPOSTTask("/tasks/delete-old-feed", values))
}
if len(tasks) > 0 {
c.Errorf("deleting %v feeds", len(tasks))
if _, err := taskqueue.AddMulti(c, tasks, ""); err != nil {
c.Errorf("err: %v", err)
}
}
if !done {
if cur, err := it.Cursor(); err == nil {
values := make(url.Values)
values.Add("c", cur.String())
taskqueue.Add(c, taskqueue.NewPOSTTask("/tasks/delete-old-feeds", values), "")
} else {
c.Errorf("err: %v", err)
}
}
}
作者:huluw
项目:gorea
func DeleteOldFeed(c mpg.Context, w http.ResponseWriter, r *http.Request) {
ctx := appengine.Timeout(c, time.Minute)
g := goon.FromContext(ctx)
oldDate := time.Now().Add(-time.Hour * 24 * 90)
feed := Feed{Url: r.FormValue("f")}
if err := g.Get(&feed); err != nil {
c.Criticalf("err: %v", err)
return
}
if feed.LastViewed.After(oldDate) {
return
}
q := datastore.NewQuery(g.Kind(&Story{})).Ancestor(g.Key(&feed)).KeysOnly()
keys, err := q.GetAll(ctx, nil)
if err != nil {
c.Criticalf("err: %v", err)
return
}
q = datastore.NewQuery(g.Kind(&StoryContent{})).Ancestor(g.Key(&feed)).KeysOnly()
sckeys, err := q.GetAll(ctx, nil)
if err != nil {
c.Criticalf("err: %v", err)
return
}
keys = append(keys, sckeys...)
c.Infof("delete: %v - %v", feed.Url, len(keys))
feed.NextUpdate = timeMax.Add(time.Hour)
if _, err := g.Put(&feed); err != nil {
c.Criticalf("put err: %v", err)
}
if len(keys) == 0 {
return
}
err = g.DeleteMulti(keys)
if err != nil {
c.Criticalf("err: %v", err)
}
}
作者:huluw
项目:gorea
func DeleteBlobs(c mpg.Context, w http.ResponseWriter, r *http.Request) {
ctx := appengine.Timeout(c, time.Minute)
q := datastore.NewQuery("__BlobInfo__").KeysOnly()
it := q.Run(ctx)
wg := sync.WaitGroup{}
something := false
for _i := 0; _i < 20; _i++ {
var bk []appengine.BlobKey
for i := 0; i < 1000; i++ {
k, err := it.Next(nil)
if err == datastore.Done {
break
} else if err != nil {
c.Errorf("err: %v", err)
continue
}
bk = append(bk, appengine.BlobKey(k.StringID()))
}
if len(bk) == 0 {
break
}
go func(bk []appengine.BlobKey) {
something = true
c.Errorf("deleteing %v blobs", len(bk))
err := blobstore.DeleteMulti(ctx, bk)
if err != nil {
c.Errorf("blobstore delete err: %v", err)
}
wg.Done()
}(bk)
wg.Add(1)
}
wg.Wait()
if something {
taskqueue.Add(c, taskqueue.NewPOSTTask("/tasks/delete-blobs", nil), "")
}
}
作者:nicksnyde
项目:nuke-appengine-datastor
func nukeDatastore(w http.ResponseWriter, r *http.Request) {
c := appengine.Timeout(appengine.NewContext(r), 1*time.Hour)
kind := r.FormValue("kind")
c.Debugf("starting query")
q := datastore.NewQuery(kind).KeysOnly().Limit(-1)
keys, err := q.GetAll(c, nil)
if err != nil {
c.Errorf("query failed: %s", err)
return
}
c.Debugf("query finished %d", len(keys))
var wg sync.WaitGroup
count := len(keys)
batchSize := 500
for i := 0; i < count; i += batchSize {
wg.Add(1)
j := i + batchSize
if j > count {
j = count
}
c.Infof("will delete keys [%d:%d]", i, j)
go func(i, j int) {
defer wg.Done()
c.Infof("starting delete [%d:%d]", i, j)
if err := datastore.DeleteMulti(c, keys[i:j]); err != nil {
c.Errorf("failed deleting keys [%d:%d]: %s", i, j, err)
} else {
c.Infof("deleted keys [%d:%d]", i, j)
}
}(i, j)
}
c.Debugf("waiting...")
wg.Wait()
w.Write([]byte(fmt.Sprintf("nuked %s", kind)))
}
作者:hugo
项目:complaint
func zipHandler(w http.ResponseWriter, r *http.Request) {
if r.FormValue("date") == "" {
var params = map[string]interface{}{
"Yesterday": date.NowInPdt().AddDate(0, 0, -1),
}
if err := templates.ExecuteTemplate(w, "zip-report-form", params); err != nil {
http.Error(w, err.Error(), http.StatusInternalServerError)
}
return
}
ctx := appengine.Timeout(appengine.NewContext(r), 60*time.Second) // Default has a 5s timeout
cdb := complaintdb.ComplaintDB{C: ctx, Memcache: false}
zip := r.FormValue("zip")
s, e, _ := widget.FormValueDateRange(r)
var countsByHour [24]int
countsByDate := map[string]int{}
var uniquesByHour [24]map[string]int
uniquesByDate := map[string]map[string]int{}
uniquesAll := map[string]int{}
iter := cdb.NewIter(cdb.QueryInSpanInZip(s, e, zip))
for {
c, err := iter.NextWithErr()
if err != nil {
http.Error(w, fmt.Sprintf("Zip iterator failed: %v", err), http.StatusInternalServerError)
return
} else if c == nil {
break // We've hit EOF
}
h := c.Timestamp.Hour()
countsByHour[h]++
if uniquesByHour[h] == nil {
uniquesByHour[h] = map[string]int{}
}
uniquesByHour[h][c.Profile.EmailAddress]++
d := c.Timestamp.Format("2006.01.02")
countsByDate[d]++
if uniquesByDate[d] == nil {
uniquesByDate[d] = map[string]int{}
}
uniquesByDate[d][c.Profile.EmailAddress]++
uniquesAll[c.Profile.EmailAddress]++
}
dateKeys := []string{}
for k, _ := range countsByDate {
dateKeys = append(dateKeys, k)
}
sort.Strings(dateKeys)
data := [][]string{}
data = append(data, []string{"Date", "NumComplaints", "UniqueComplainers"})
for _, k := range dateKeys {
data = append(data, []string{
k,
fmt.Sprintf("%d", countsByDate[k]),
fmt.Sprintf("%d", len(uniquesByDate[k])),
})
}
data = append(data, []string{"------"})
data = append(data, []string{"HourAcrossAllDays", "NumComplaints", "UniqueComplainers"})
for i, v := range countsByHour {
data = append(data, []string{
fmt.Sprintf("%02d:00", i),
fmt.Sprintf("%d", v),
fmt.Sprintf("%d", len(uniquesByHour[i])),
})
}
data = append(data, []string{"------"})
data = append(data, []string{"UniqueComplainersAcrossAllDays", fmt.Sprintf("%d", len(uniquesAll))})
var params = map[string]interface{}{"Data": data}
if err := templates.ExecuteTemplate(w, "report", params); err != nil {
http.Error(w, err.Error(), http.StatusInternalServerError)
}
}
作者:hugo
项目:complaint
func personalReportHandler(w http.ResponseWriter, r *http.Request) {
session := sessions.Get(r)
if session.Values["email"] == nil {
http.Error(w, "session was empty; no cookie ?", http.StatusInternalServerError)
return
}
email := session.Values["email"].(string)
if r.FormValue("date") == "" {
var params = map[string]interface{}{
"Yesterday": date.NowInPdt().AddDate(0, 0, -1),
}
if err := templates.ExecuteTemplate(w, "personal-report-form", params); err != nil {
http.Error(w, err.Error(), http.StatusInternalServerError)
}
return
}
start, end, _ := widget.FormValueDateRange(r)
ctx := appengine.Timeout(appengine.NewContext(r), 60*time.Second)
cdb := complaintdb.ComplaintDB{C: ctx}
w.Header().Set("Content-Type", "text/plain")
// w.Header().Set("Content-Disposition", fmt.Sprintf("attachment; filename=\"%s\"", "sc.txt"))
fmt.Fprintf(w, "Personal disturbances report for <%s>:\n From [%s]\n To [%s]\n",
email, start, end)
complaintStrings := []string{}
var countsByHour [24]int
countsByDate := map[string]int{}
countsByAirline := map[string]int{}
iter := cdb.NewIter(cdb.QueryInSpanByEmailAddress(start, end, email))
n := 0
for {
c := iter.Next()
if c == nil {
break
}
str := fmt.Sprintf("Time: %s, Loudness:%d, Speedbrakes:%v, Flight:%6.6s, Notes:%s",
c.Timestamp.Format("2006.01.02 15:04:05"), c.Loudness, c.HeardSpeedbreaks,
c.AircraftOverhead.FlightNumber, c.Description)
n++
complaintStrings = append(complaintStrings, str)
countsByHour[c.Timestamp.Hour()]++
countsByDate[c.Timestamp.Format("2006.01.02")]++
if airline := c.AircraftOverhead.IATAAirlineCode(); airline != "" {
countsByAirline[airline]++
}
}
fmt.Fprintf(w, "\nTotal number of disturbance reports, over %d days: %d\n",
len(countsByDate), n)
fmt.Fprintf(w, "\nDisturbance reports, counted by Airline (where known):\n")
for _, k := range keysByIntValDesc(countsByAirline) {
fmt.Fprintf(w, " %s: % 4d\n", k, countsByAirline[k])
}
fmt.Fprintf(w, "\nDisturbance reports, counted by date:\n")
for _, k := range keysByKeyAsc(countsByDate) {
fmt.Fprintf(w, " %s: % 4d\n", k, countsByDate[k])
}
fmt.Fprintf(w, "\nDisturbance reports, counted by hour of day (across all dates):\n")
for i, n := range countsByHour {
fmt.Fprintf(w, " %02d: % 4d\n", i, n)
}
fmt.Fprintf(w, "\nFull dump of all disturbance reports:\n\n")
for _, s := range complaintStrings {
fmt.Fprint(w, s+"\n")
}
}
作者:hugo
项目:complaint
func monthTaskHandler(w http.ResponseWriter, r *http.Request) {
//ctx,_ := context.WithTimeout(appengine.NewContext(r), 599*time.Second)
ctx := appengine.NewContext(r)
cdb := complaintdb.ComplaintDB{
//C: oldappengine.NewContext(r),
C: oldappengine.Timeout(oldappengine.NewContext(r), 599*time.Second),
}
year, err := strconv.ParseInt(r.FormValue("year"), 10, 64)
if err != nil {
http.Error(w, "need arg 'year' (2015)", http.StatusInternalServerError)
return
}
month, err := strconv.ParseInt(r.FormValue("month"), 10, 64)
if err != nil {
http.Error(w, "need arg 'month' (1-12)", http.StatusInternalServerError)
return
}
now := date.NowInPdt()
s := time.Date(int(year), time.Month(month), 1, 0, 0, 0, 0, now.Location())
e := s.AddDate(0, 1, 0).Add(-1 * time.Second)
log.Infof(ctx, "Starting /be/month: %s", s)
// One time, at 00:00, for each day of the given month
days := date.IntermediateMidnights(s.Add(-1*time.Second), e)
filename := s.Format("complaints-20060102") + e.Format("-20060102.csv")
gcsHandle, err := gcs.OpenRW(ctx, "serfr0-reports", filename, "text/plain")
if err != nil {
http.Error(w, err.Error(), http.StatusInternalServerError)
return
}
csvWriter := csv.NewWriter(gcsHandle.IOWriter())
cols := []string{
"CallerCode", "Name", "Address", "Zip", "Email", "HomeLat", "HomeLong",
"UnixEpoch", "Date", "Time(PDT)", "Notes", "ActivityDisturbed", "Flightnumber",
"Notes",
// Column names above are incorrect, but BKSV are used to them.
//
//"CallerCode", "Name", "Address", "Zip", "Email", "HomeLat", "HomeLong",
//"UnixEpoch", "Date", "Time(PDT)", "Notes", "Flightnumber",
//"ActivityDisturbed", "CcSFO",
}
csvWriter.Write(cols)
for _, dayStart := range days {
dayEnd := dayStart.AddDate(0, 0, 1).Add(-1 * time.Second)
log.Infof(ctx, " /be/month: %s - %s", dayStart, dayEnd)
iter := cdb.NewIter(cdb.QueryInSpan(dayStart, dayEnd))
for {
c, err := iter.NextWithErr()
if err != nil {
http.Error(w, fmt.Sprintf("iterator failed: %v", err),
http.StatusInternalServerError)
return
}
if c == nil {
break
}
r := []string{
c.Profile.CallerCode,
c.Profile.FullName,
c.Profile.Address,
c.Profile.StructuredAddress.Zip,
c.Profile.EmailAddress,
fmt.Sprintf("%.4f", c.Profile.Lat),
fmt.Sprintf("%.4f", c.Profile.Long),
fmt.Sprintf("%d", c.Timestamp.UTC().Unix()),
c.Timestamp.Format("2006/01/02"),
c.Timestamp.Format("15:04:05"),
c.Description,
c.AircraftOverhead.FlightNumber,
c.Activity,
fmt.Sprintf("%v", c.Profile.CcSfo),
}
if err := csvWriter.Write(r); err != nil {
http.Error(w, err.Error(), http.StatusInternalServerError)
return
}
}
}
csvWriter.Flush()
if err := gcsHandle.Close(); err != nil {
http.Error(w, err.Error(), http.StatusInternalServerError)
return
}
log.Infof(ctx, "GCS report '%s' successfully written", filename)
w.Header().Set("Content-Type", "text/plain")
w.Write([]byte(fmt.Sprintf("OK!\nGCS file '%s' written to bucket", filename)))
//.........这里部分代码省略.........
作者:JesseLivingsto
项目:gorea
func ListFeeds(c mpg.Context, w http.ResponseWriter, r *http.Request) {
cu := user.Current(c)
gn := goon.FromContext(c)
u := &User{Id: cu.ID}
ud := &UserData{Id: "data", Parent: gn.Key(u)}
if err := gn.GetMulti([]interface{}{u, ud}); err != nil && !goon.NotFound(err, 1) {
serveError(w, err)
return
}
l := &Log{
Parent: ud.Parent,
Id: time.Now().UnixNano(),
Text: "list feeds",
}
l.Text += fmt.Sprintf(", len opml %v", len(ud.Opml))
putU := false
putUD := false
fixRead := false
if time.Since(u.Read) > oldDuration {
u.Read = time.Now().Add(-oldDuration)
putU = true
fixRead = true
l.Text += ", u.Read"
}
trialRemaining := 0
if STRIPE_KEY != "" && ud.Opml != nil {
if u.Created.IsZero() {
u.Created = time.Now()
putU = true
} else if time.Since(u.Created) > accountFreeDuration {
b, _ := json.Marshal(struct {
ErrorSubscription bool
}{
true,
})
w.Write(b)
return
}
trialRemaining = int((accountFreeDuration-time.Since(u.Created))/time.Hour/24) + 1
}
read := make(Read)
var uf Opml
c.Step("unmarshal user data", func(c mpg.Context) {
gob.NewDecoder(bytes.NewReader(ud.Read)).Decode(&read)
json.Unmarshal(ud.Opml, &uf)
})
var feeds []*Feed
opmlMap := make(map[string]*OpmlOutline)
var merr error
c.Step("fetch feeds", func(c mpg.Context) {
gn := goon.FromContext(appengine.Timeout(c, time.Minute))
for _, outline := range uf.Outline {
if outline.XmlUrl == "" {
for _, so := range outline.Outline {
feeds = append(feeds, &Feed{Url: so.XmlUrl})
opmlMap[so.XmlUrl] = so
}
} else {
feeds = append(feeds, &Feed{Url: outline.XmlUrl})
opmlMap[outline.XmlUrl] = outline
}
}
merr = gn.GetMulti(feeds)
})
lock := sync.Mutex{}
fl := make(map[string][]*Story)
q := datastore.NewQuery(gn.Key(&Story{}).Kind()).
Filter(IDX_COL+" >=", u.Read).
KeysOnly().
Order("-" + IDX_COL).
Limit(250)
updatedLinks := false
now := time.Now()
numStories := 0
c.Step(fmt.Sprintf("feed unreads: %v", u.Read), func(c mpg.Context) {
queue := make(chan *Feed)
tc := make(chan *taskqueue.Task)
done := make(chan bool)
wg := sync.WaitGroup{}
feedProc := func() {
for f := range queue {
c.Step(f.Title, func(c mpg.Context) {
defer wg.Done()
var stories []*Story
gn := goon.FromContext(appengine.Timeout(c, time.Minute))
if !f.Date.Before(u.Read) {
fk := gn.Key(f)
sq := q.Ancestor(fk)
keys, _ := gn.GetAll(sq, nil)
stories = make([]*Story, len(keys))
for j, key := range keys {
stories[j] = &Story{
Id: key.StringID(),
Parent: fk,
}
}
gn.GetMulti(stories)
}
//.........这里部分代码省略.........