Merge pull request datatogether#2 from datatogether/custom_crawls

Add support for `/customcrawls` endpoint

custom_crawl_handlers.go

@@ -0,0 +1,91 @@
+package main
+
+import (
+	"encoding/json"
+	"github.com/datatogether/api/apiutil"
+	"github.com/datatogether/archive"
+	"net/http"
+)
+
+func CustomCrawlHandler(w http.ResponseWriter, r *http.Request) {
+	switch r.Method {
+	case "OPTIONS":
+		EmptyOkHandler(w, r)
+	case "GET":
+		GetCustomCrawlHandler(w, r)
+	case "PUT":
+		SaveCustomCrawlHandler(w, r)
+	case "DELETE":
+		DeleteCustomCrawlHandler(w, r)
+	default:
+		NotFoundHandler(w, r)
+	}
+}
+
+func CustomCrawlsHandler(w http.ResponseWriter, r *http.Request) {
+	switch r.Method {
+	case "GET":
+		ListCustomCrawlsHandler(w, r)
+	case "PUT", "POST":
+		SaveCustomCrawlHandler(w, r)
+	default:
+		NotFoundHandler(w, r)
+	}
+}
+
+func GetCustomCrawlHandler(w http.ResponseWriter, r *http.Request) {
+	res := &archive.CustomCrawl{}
+	args := &CustomCrawlsGetParams{
+		Id: r.URL.Path[len("/customcrawls/"):],
+	}
+	err := new(CustomCrawls).Get(args, res)
+	if err != nil {
+		apiutil.WriteErrResponse(w, http.StatusInternalServerError, err)
+		return
+	}
+	apiutil.WriteResponse(w, res)
+}
+
+func ListCustomCrawlsHandler(w http.ResponseWriter, r *http.Request) {
+	p := apiutil.PageFromRequest(r)
+	res := make([]*archive.CustomCrawl, p.Size)
+	args := &CustomCrawlsListParams{
+		Limit:   p.Limit(),
+		Offset:  p.Offset(),
+		OrderBy: "created",
+	}
+	err := new(CustomCrawls).List(args, &res)
+	if err != nil {
+		apiutil.WriteErrResponse(w, http.StatusInternalServerError, err)
+		return
+	}
+	apiutil.WritePageResponse(w, res, r, p)
+}
+
+func SaveCustomCrawlHandler(w http.ResponseWriter, r *http.Request) {
+	un := &archive.CustomCrawl{}
+	if err := json.NewDecoder(r.Body).Decode(un); err != nil {
+		apiutil.WriteErrResponse(w, http.StatusInternalServerError, err)
+		return
+	}
+	res := &archive.CustomCrawl{}
+	if err := new(CustomCrawls).Save(un, res); err != nil {
+		apiutil.WriteErrResponse(w, http.StatusInternalServerError, err)
+		return
+	}
+	apiutil.WriteResponse(w, res)
+}
+
+func DeleteCustomCrawlHandler(w http.ResponseWriter, r *http.Request) {
+	un := &archive.CustomCrawl{}
+	if err := json.NewDecoder(r.Body).Decode(un); err != nil {
+		apiutil.WriteErrResponse(w, http.StatusInternalServerError, err)
+		return
+	}
+	res := &archive.CustomCrawl{}
+	if err := new(CustomCrawls).Save(un, res); err != nil {
+		apiutil.WriteErrResponse(w, http.StatusInternalServerError, err)
+		return
+	}
+	apiutil.WriteResponse(w, res)
+}

custom_crawl_requests.go

@@ -0,0 +1,59 @@
+package main
+
+import (
+	"github.com/datatogether/archive"
+)
+
+type CustomCrawls int
+
+type CustomCrawlsGetParams struct {
+	Id string
+}
+
+func (u *CustomCrawls) Get(p *CustomCrawlsGetParams, res *archive.CustomCrawl) (err error) {
+	url := &archive.CustomCrawl{
+		Id: p.Id,
+	}
+	err = url.Read(store)
+	if err != nil {
+		return err
+	}
+
+	*res = *url
+	return nil
+}
+
+type CustomCrawlsListParams struct {
+	OrderBy string
+	Limit   int
+	Offset  int
+}
+
+func (u *CustomCrawls) List(p *CustomCrawlsListParams, res *[]*archive.CustomCrawl) (err error) {
+	urls, err := archive.ListCustomCrawls(store, p.Limit, p.Offset)
+	if err != nil {
+		return err
+	}
+	*res = urls
+	return nil
+}
+
+func (u *CustomCrawls) Save(model *archive.CustomCrawl, res *archive.CustomCrawl) (err error) {
+	err = model.Save(store)
+	if err != nil {
+		return err
+	}
+
+	*res = *model
+	return nil
+}
+
+func (u *CustomCrawls) Delete(model *archive.CustomCrawl, res *archive.CustomCrawl) (err error) {
+	err = model.Delete(store)
+	if err != nil {
+		return err
+	}
+
+	*res = *model
+	return nil
+}

server.go

@@ -102,6 +102,9 @@ func NewServerRoutes() *http.ServeMux {
 	m.Handle("/uncrawlables", middleware(UncrawlablesHandler))
 	m.Handle("/uncrawlables/", middleware(UncrawlableHandler))
 
+	m.Handle("/customcrawls", middleware(CustomCrawlsHandler))
+	m.Handle("/customcrawls/", middleware(CustomCrawlHandler))
+
 	m.HandleFunc("/.well-known/acme-challenge/", CertbotHandler)
 
 	return m
@@ -138,6 +141,7 @@ func initPostgres() {
 		&archive.Primer{},
 		&archive.Source{},
 		&archive.Uncrawlable{},
+		&archive.CustomCrawl{},
 		&archive.Url{},
 	)
 }