Commit 0e8e3cd1 authored by Andreas Wagner's avatar Andreas Wagner
Browse files

File upload working.

parent 131fa9b2
......@@ -2,14 +2,16 @@ package routing
import (
"bytes"
"fmt"
"log"
"net/http"
"strings"
"github.com/gin-contrib/cors"
"github.com/gin-gonic/gin"
"github.com/stephenmuss/ginerus"
"gitlab.gwdg.de/rg-mpg-de/tei2zenodo"
"gitlab.gwdg.de/rg-mpg-de/tei2zenodo/internal/pkg/xml"
"gitlab.gwdg.de/rg-mpg-de/tei2zenodo/internal/pkg/t2zxml"
"gitlab.gwdg.de/rg-mpg-de/tei2zenodo/internal/pkg/zenodo"
)
......@@ -33,60 +35,80 @@ func SetupRoutes(conf tei2zenodo.Config) *gin.Engine {
// Read file from Request body
buf := new(bytes.Buffer)
buf.ReadFrom(c.Request.Body)
// log.Printf("Here is the file: %s", buf)
file := buf.String()
r := strings.NewReader(file)
// Get filename for upload
filename := c.Request.FormValue(`filename`)
if filename == "" {
filename = t2zxml.GetFilename(r)
r.Seek(0, 0)
}
log.Printf("Beginning of submitted file %s: %s ...", filename, file[:100])
// Parse TEI file
log.Printf("=== Parse TEI file ===")
mc := conf.Metadata
var md tei2zenodo.ZMetadata
doi, err := xml.ParseTEI(buf, &md, &mc)
// var doc *t2zxml.TEIDoc
// doc, doi, err := t2zxml.ParseTEI(r, &md, &mc)
_, doi, err := t2zxml.ParseTEI(r, &md, &mc)
if err != nil {
log.Printf("Error (%s) parsing TEI file: %v", err, buf)
log.Printf("Error (%s) parsing TEI file: %80s", err, file)
AbortMsg(500, err, c)
}
r.Seek(0, 0)
switch doi {
case "":
// prereserve doi
err := zenodo.GetDOI(buf, &md)
if err != nil {
log.Printf("--- TEI successfully parsed ---")
log.Printf("Title: %v", md.Title)
log.Printf("Creator 1: %v", md.Creators[0])
log.Printf("Contributor 1: %v", md.Contributors[0])
if doi != "" {
log.Printf("DOI present: %v", doi)
} else {
log.Printf("=== Fetching DOI from zenodo ===")
doi, err := zenodo.GetDOI(&md, &conf.Repo)
if err != nil || doi == "" {
log.Printf("Error creating DOI reservation deposit: %v", err)
AbortMsg(500, err, c)
AbortMsg(500, fmt.Errorf("problem reported by zenodo: %s", err), c)
} else {
log.Printf("New DOI minted: %s", doi)
}
// If we have any, stream laws
// timestamp := time.Now().UTC().Format("(2006-01-02_15-04-05)")
// c.Writer.Header().Set("Content-Disposition", "attachment; filename="+downloadName)
c.Writer.Header().Set("Content-Description", "File Transfer")
c.Writer.Header().Set("Access-Control-Allow-Origin", "*")
c.Writer.Header().Set("Access-Control-Expose-Headers", "Content-Disposition")
c.Writer.Header().Set("X-Accel-Buffering", "no")
c.Writer.Header().Set("Content-type", "text/xml")
md.DOI = doi
/*
err := zenodo.PostFile(buf, %md)
if err != nil {
log.Printf("Error posting file: %v", err)
AbortMsg(500, err, c)
}
err := zenodo.PutMetadata(buf, %md)
if err != nil {
log.Printf("Error posting file: %v", err)
AbortMsg(500, err, c)
}
err := zenodo.Publish(buf, %md)
log.Printf("=== Adding new DOI to document ===")
err = t2zxml.MixinDOI(&doc, doi)
if err != nil {
log.Printf("Error posting file: %v", err)
AbortMsg(500, err, c)
log.Printf("Error mixing new DOI into document: %v", err)
AbortMsg(500, fmt.Errorf("problem mixing new DOI into document: %s", err), c)
}
*/
default:
log.Printf("--- TEI successfully parsed ---")
log.Printf("DOI present: %v", doi)
log.Printf("Title: %v", md.Title)
log.Printf("Contributor 1: %v", md.Contributors[0])
log.Printf("Contributor 2: %v", md.Contributors[1])
}
r.Seek(0, 0)
log.Printf("=== Sending to zenodo ===")
err = zenodo.PostFile(r, filename, &md, &conf.Repo)
if err != nil {
log.Printf("Error sending POST request to zenodo: %v", err)
AbortMsg(500, err, c)
}
/*
err := zenodo.PutMetadata(buf, %md)
if err != nil {
log.Printf("Error posting file: %v", err)
AbortMsg(500, err, c)
}
err := zenodo.Publish(buf, %md)
if err != nil {
log.Printf("Error posting file: %v", err)
AbortMsg(500, err, c)
}
*/
})
APIv1.GET("/webhook", func(c *gin.Context) {
})
......
package xml
package t2zxml
import (
"bytes"
"fmt"
"io"
"log"
"reflect"
"regexp"
......@@ -12,15 +12,37 @@ import (
"gitlab.gwdg.de/rg-mpg-de/tei2zenodo"
)
// TEIDoc represents a parsed TEI XML file
type TEIDoc xmlquery.Node
// GetFilename extracts what can serve as filename. Like /TEI/@xml:id
func GetFilename(r io.Reader) string {
// Parse document (in r) wih antchfx/xmlquery...
doc, err := xmlquery.Parse(r)
if err != nil {
log.Printf("Could not parse xml.\n")
return ""
}
t := xmlquery.FindOne(doc, `/TEI/@xml:id`)
if n := t; t != nil {
u := n.InnerText()
return u + ".xml"
} else {
return ""
}
}
// ParseTEI reads a TEI file and parses its metadata into a ZMetadata variable.
// Returns a doi (maybe empty) and an error value.
func ParseTEI(buf *bytes.Buffer, md *tei2zenodo.ZMetadata, conf *tei2zenodo.MetadataConfig) (string, error) {
func ParseTEI(r io.Reader, md *tei2zenodo.ZMetadata, conf *tei2zenodo.MetadataConfig) (*xmlquery.Node, string, error) {
// Parse document (in buf) wih antchfx/xmlquery...
doc, err := xmlquery.Parse(buf)
var doc *xmlquery.Node
// Parse document (in r) wih antchfx/xmlquery...
doc, err := xmlquery.Parse(r)
if err != nil {
log.Printf("Could not parse xml.\n")
return "", err
return doc, "", err
}
re := regexp.MustCompile(`\s+`)
......@@ -43,7 +65,6 @@ func ParseTEI(buf *bytes.Buffer, md *tei2zenodo.ZMetadata, conf *tei2zenodo.Meta
f := s.Field(i)
structFieldname := typeOfT.Field(i).Name
structFieldtype := f.Type()
// structFieldvalue := f.Interface()
jsonFieldname := typeOfT.Field(i).Tag.Get("json")
for j := range conf.Fields {
if conf.Fields[j].Field == jsonFieldname {
......@@ -64,15 +85,10 @@ func ParseTEI(buf *bytes.Buffer, md *tei2zenodo.ZMetadata, conf *tei2zenodo.Meta
varType := structFieldtype.Elem()
zc := reflect.Indirect(reflect.New(varType))
// u := tei2zenodo.ZContributor{Name: string(i), Type: "Test Role"}
// v := &u
// zc := reflect.ValueOf(v).Elem()
// typeOfZcT := zc.Type()
for m := 0; m < zc.NumField(); m++ {
zcf := zc.Field(m)
zStructFieldname := varType.Field(m).Name
// zStructFieldname := varType.Field(m).Name
zStructFieldtype := zcf.Type()
// ztructRFieldvalue := rf.Interface()
zJSONFieldname := varType.Field(m).Tag.Get("json")
for m := range confSubfields {
if confSubfields[m].Field == zJSONFieldname {
......@@ -83,9 +99,8 @@ func ParseTEI(buf *bytes.Buffer, md *tei2zenodo.ZMetadata, conf *tei2zenodo.Meta
var u string
t := xmlquery.FindOne(l, xrpath)
if n := t; t != nil {
// u = strings.Replace(strings.Replace(strings.TrimSpace(n.InnerText()), "\n", " ", -1), " ", " ", -1)
u = re.ReplaceAllString(n.InnerText(), " ")
log.Printf("%s.%s found. Set to '%v' ...\n", structFieldname, zStructFieldname, u)
// log.Printf("%s.%s found. Set to '%v' ...\n", structFieldname, zStructFieldname, u)
zcf.SetString(u)
}
} else if confSubfields[m].XExpression != "" {
......@@ -93,7 +108,7 @@ func ParseTEI(buf *bytes.Buffer, md *tei2zenodo.ZMetadata, conf *tei2zenodo.Meta
xexpr, err := xpath.Compile(confSubfields[m].XExpression)
if err != nil {
log.Printf("Erroneous XPath expression: %s ...", confSubfields[m].XExpression)
return "", fmt.Errorf("unknown (hardcoded?) metadata type: %s.%s", structFieldtype, zStructFieldtype)
return doc, "", fmt.Errorf("unknown (hardcoded?) metadata type: %s.%s", structFieldtype, zStructFieldtype)
}
switch zStructFieldtype.String() {
case "string":
......@@ -101,7 +116,7 @@ func ParseTEI(buf *bytes.Buffer, md *tei2zenodo.ZMetadata, conf *tei2zenodo.Meta
// TODO: Check for wrong types (e.g. xepr returning int instead of string)
u = xexpr.Evaluate(xmlquery.CreateXPathNavigator(l)).(string)
if u != "" {
log.Printf("%s.%s found. Set to '%v' ...\n", structFieldname, zStructFieldname, u)
// log.Printf("%s.%s found. Set to '%v' ...\n", structFieldname, zStructFieldname, u)
zcf.SetString(u)
}
......@@ -114,16 +129,16 @@ func ParseTEI(buf *bytes.Buffer, md *tei2zenodo.ZMetadata, conf *tei2zenodo.Meta
newSlice := reflect.Append(f, reflect.ValueOf(n))
zcf.Set(newSlice)
}
log.Printf("%s.%s found. Set to '%v' ...\n", structFieldname, zStructFieldname, u)
// log.Printf("%s.%s found. Set to '%v' ...\n", structFieldname, zStructFieldname, u)
default:
log.Printf("Unknown (hardcoded?) metadata type: %s.%s ...", structFieldtype, zStructFieldtype)
return "", fmt.Errorf("xml: unknown (hardcoded?) metadata type: %s.%s", structFieldtype, zStructFieldtype)
return doc, "", fmt.Errorf("xml: unknown (hardcoded?) metadata type: %s.%s", structFieldtype, zStructFieldtype)
}
} else if confSubfields[m].Field == "name" || confSubfields[m].Field == "type" {
log.Printf("Problem with config: XPath or XExpression missing in %v ...", conf.Fields[j])
return "", fmt.Errorf("xml: malformed config (xpath/xexpression missing): %v", conf.Fields[j])
return doc, "", fmt.Errorf("xml: malformed config (xpath/xexpression missing): %v", conf.Fields[j])
}
}
}
......@@ -155,14 +170,12 @@ func ParseTEI(buf *bytes.Buffer, md *tei2zenodo.ZMetadata, conf *tei2zenodo.Meta
f.Set(newSlice)
default:
log.Printf("Problem with type conversion of %s (%s)", structFieldname, varType.Name())
return "", fmt.Errorf("xml: malformed config (type problem in %s [%s])", structFieldname, varType.Name())
return doc, "", fmt.Errorf("xml: malformed config (type problem in %s [%s])", structFieldname, varType.Name())
}
// newSlice := reflect.Append(f, reflect.ValueOf(zc))
// f.Set(newSlice)
}
} else {
log.Printf("Problem with config: XPath missing in %v ...", conf.Fields[j])
return "", fmt.Errorf("xml: malformed config (xpath missing): %v", conf.Fields[j])
return doc, "", fmt.Errorf("xml: malformed config (xpath missing): %v", conf.Fields[j])
}
} else if conf.Fields[j].XPath != "" {
......@@ -172,32 +185,30 @@ func ParseTEI(buf *bytes.Buffer, md *tei2zenodo.ZMetadata, conf *tei2zenodo.Meta
var u string
t := xmlquery.FindOne(doc, xpath)
if n := t; t != nil {
// u = strings.Replace(strings.Replace(strings.TrimSpace(n.InnerText()), "\n", " ", -1), " ", " ", -1)
u = re.ReplaceAllString(n.InnerText(), " ")
log.Printf("%s found. Set to '%v' ...\n", structFieldname, u)
// log.Printf("%s found. Set to '%v' ...\n", structFieldname, u)
f.SetString(u)
}
case "[]string":
var u []string
for _, n := range xmlquery.Find(doc, xpath) {
// v = append(u, strings.Replace(strings.Replace(strings.TrimSpace(n.InnerText()), "\n", " ", -1), " ", " ", -1))
v := re.ReplaceAllString(n.InnerText(), " ")
u = append(u, v)
newSlice := reflect.Append(f, reflect.ValueOf(v))
f.Set(newSlice)
}
log.Printf("%s found. Set to '%v' ...\n", structFieldname, u)
// log.Printf("%s found. Set to '%v' ...\n", structFieldname, u)
default:
log.Printf("Unknown (hardcoded?) metadata type: %s ...", structFieldtype)
return "", fmt.Errorf("xml: unknown (hardcoded?) metadata type: %s", structFieldtype)
return doc, "", fmt.Errorf("xml: unknown (hardcoded?) metadata type: %s", structFieldtype)
}
} else if conf.Fields[j].XExpression != "" {
xexpr, err := xpath.Compile(conf.Fields[j].XExpression)
if err != nil {
log.Printf("Erroneous XPath expression: %s ...", conf.Fields[j].XExpression)
return "", fmt.Errorf("unknown (hardcoded?) metadata type: %s", structFieldtype)
return doc, "", fmt.Errorf("unknown (hardcoded?) metadata type: %s", structFieldtype)
}
switch structFieldtype.String() {
case "string":
......@@ -205,7 +216,7 @@ func ParseTEI(buf *bytes.Buffer, md *tei2zenodo.ZMetadata, conf *tei2zenodo.Meta
// TODO: Check for wrong types (e.g. xepr returning int instead of string)
u = xexpr.Evaluate(xmlquery.CreateXPathNavigator(doc)).(string)
if u != "" {
log.Printf("%s found. Set to '%v' ...\n", structFieldname, u)
// log.Printf("%s found. Set to '%v' ...\n", structFieldname, u)
f.SetString(u)
}
......@@ -218,20 +229,25 @@ func ParseTEI(buf *bytes.Buffer, md *tei2zenodo.ZMetadata, conf *tei2zenodo.Meta
newSlice := reflect.Append(f, reflect.ValueOf(n))
f.Set(newSlice)
}
log.Printf("%s found. Set to '%v' ...\n", structFieldname, u)
// log.Printf("%s found. Set to '%v' ...\n", structFieldname, u)
default:
log.Printf("Unknown (hardcoded?) metadata type: %s ...", structFieldtype)
return "", fmt.Errorf("xml: unknown (hardcoded?) metadata type: %s", structFieldtype)
return doc, "", fmt.Errorf("xml: unknown (hardcoded?) metadata type: %s", structFieldtype)
}
} else {
log.Printf("Malformed config entry: %v ...", conf.Fields[j])
return "", fmt.Errorf("xml: malformed config entry: %v", conf.Fields[j])
return doc, "", fmt.Errorf("xml: malformed config entry: %v", conf.Fields[j])
}
}
}
}
doi := md.DOI
return doi, nil
return doc, doi, nil
}
// MixinDOI adds a DOI idno element to the document
func MixinDOI(doc *xmlquery.Node, doi string) error {
return nil
}
......@@ -2,53 +2,219 @@ package zenodo
import (
"bytes"
"encoding/json"
"fmt"
"io"
"io/ioutil"
"log"
"mime/multipart"
"net/http"
"strconv"
"strings"
"gitlab.gwdg.de/rg-mpg-de/tei2zenodo"
)
// ZResponse stores a zenodo response
type ZResponse struct {
Created string
Files []string
ID int64
Links ZLinks
Metadata ZMetadata
Modified string
Owner int64
RecordID int64 `json:"record_id"`
State string
Submitted bool
Title string
}
// ZLinks stores links to zenodo's REST endpoints
type ZLinks struct {
Discard string
Edit string
Files string
Publish string
NewVersion string
Self string
}
// ZMetadata stores zenodo metadata
type ZMetadata struct {
PrereserveDOI ZPrereserveDOI `json:"prereserve_doi"`
}
// ZPrereserveDOI stores prereservation information
type ZPrereserveDOI struct {
DOI string
RecID int64
}
// GetDOI creates an empty zenodo deposit and reserves a DOI
func GetDOI(buf *bytes.Buffer, md *tei2zenodo.ZMetadata) error {
resp, err := http.Post("http://example.com/upload", "image/jpeg", buf)
func GetDOI(md *tei2zenodo.ZMetadata, conf *tei2zenodo.RepoConfig) (string, error) {
uri := conf.Host + ":" + strconv.Itoa(int(conf.Port)) + "/api/deposit/depositions?access_token=" + conf.Token
log.Printf("Post request to: %s", uri)
client := &http.Client{}
v := []byte(`{}`)
buf := bytes.NewBuffer(v)
req, err := http.NewRequest("POST", uri, buf)
if err != nil {
// handle error
log.Printf("Problem creating POST request: %v ...", err)
return "", fmt.Errorf("problem creating POST request: %s", err)
}
req.Header.Add("Content-Type", `application/json`)
resp, err := client.Do(req)
if err != nil {
log.Printf("Problem sending POST request: %v ...", err)
return "", fmt.Errorf("problem sending POST request: %s", err)
}
defer resp.Body.Close()
return nil
/* This call should return something like this, from which we want to retrieve the id:
{
"created": "2016-06-15T16:10:03.319363+00:00",
"files": [],
"id": 1234,
"links": {
"discard": "https://zenodo.org/api/deposit/depositions/1234/actions/discard",
"edit": "https://zenodo.org/api/deposit/depositions/1234/actions/edit",
"files": "https://zenodo.org/api/deposit/depositions/1234/files",
"publish": "https://zenodo.org/api/deposit/depositions/1234/actions/publish",
"newversion": "https://zenodo.org/api/deposit/depositions/1234/actions/newversion",
"self": "https://zenodo.org/api/deposit/depositions/1234"
},
"metadata": {
"prereserve_doi": {
"doi": "10.5072/zenodo.1234",
"recid": 1234
}
},
"modified": "2016-06-15T16:10:03.319371+00:00",
"owner": 1,
"record_id": 1234,
"state": "unsubmitted",
"submitted": false,
"title": ""
content, err := ioutil.ReadAll(resp.Body)
if err != nil {
log.Printf("Problem reading POST response: %v ...", err)
return "", fmt.Errorf("problem reading POST response: %s", err)
}
if strconv.Itoa(resp.StatusCode)[:1] != "2" {
log.Printf("Problem reported by zenodo: %d %v. %80s ...", resp.StatusCode, err, content)
return "", fmt.Errorf("problem reported by zenodo: %d %v. %80s", resp.StatusCode, err, content)
}
*/
var parsedContent ZResponse
err = json.Unmarshal(content, &parsedContent)
if err != nil {
log.Printf("Problem parsing zenodo's response: %v ...", err)
return "", fmt.Errorf("problem parsing zenodo's response: %s", err)
}
doi := parsedContent.Metadata.PrereserveDOI.DOI
return doi, nil
}
// PostFile posts a file to zenodo, taking the id from the md.DOI field
func PostFile(r io.Reader, filename string, md *tei2zenodo.ZMetadata, conf *tei2zenodo.RepoConfig) error {
if md.DOI[:15] != "10.5072/zenodo." { // not a zenodo doi
log.Printf("Problem: DOI %s is not a zenodo DOI.", md.DOI)
return fmt.Errorf("invalid DOI value")
} else {
id := md.DOI[15:]
// Read file from Request body
buf := new(bytes.Buffer)
buf.ReadFrom(r)
file := buf.String()
reader := strings.NewReader(file)
uri := conf.Host + ":" + strconv.Itoa(int(conf.Port)) + "/api/deposit/depositions/" + id + "/files?access_token=" + conf.Token
// uri := "https://postman-echo.com/post"
log.Printf("Post request to: %s", uri)
log.Printf("Upload filename: %s", filename)
log.Printf("Upload file: %s", file[:80])
body := new(bytes.Buffer)
writer := multipart.NewWriter(body)
label, err := writer.CreateFormField("name")
if err != nil {
log.Printf("Problem creating POST request body: %v ...", err)
return fmt.Errorf("problem creating POST request body: %s", err)
}
label.Write([]byte(filename))
part, err := writer.CreateFormFile(`file`, filename)
if err != nil {
log.Printf("Problem creating POST request body: %v ...", err)
return fmt.Errorf("problem creating POST request body: %s", err)
}
io.Copy(part, reader)
writer.Close()
req, err := http.NewRequest("POST", uri, body)
if err != nil {
log.Printf("Problem creating POST request: %v ...", err)
return fmt.Errorf("problem creating POST request: %s", err)
}
req.Header.Add(`Content-Type`, writer.FormDataContentType())
reader.Seek(0, 0)
client := &http.Client{}
resp, err := client.Do(req)
if err != nil {
log.Printf("Problem sending POST request: %v ...", err)
return fmt.Errorf("problem sending POST request: %s", err)
}
defer resp.Body.Close()
content, err := ioutil.ReadAll(resp.Body)
if err != nil {
log.Printf("Problem reading POST response: %v ...", err)
return fmt.Errorf("problem reading POST response: %s", err)
}
if strconv.Itoa(resp.StatusCode)[:1] != "2" {
log.Printf("Problem reported by zenodo: %d %v. %80s ...", resp.StatusCode, err, content)
return fmt.Errorf("problem reported by zenodo: %v. %s", err, content)
}
log.Printf("Response code %d, content: %80s", resp.StatusCode, content)
return nil
}
}
// func PostFile
// PutMetadata adds metadata to an existing deposit
func PutMetadata(md *tei2zenodo.ZMetadata, conf *tei2zenodo.RepoConfig) error {
if md.DOI[:15] != "10.5072/zenodo." { // not a zenodo doi
log.Printf("Problem: DOI is not a zenodo DOI.")
return fmt.Errorf("invalid DOI value")
} else {
id := md.DOI[15:]
mdJSON, err := json.Marshal(md)
if err != nil {
log.Printf("Problem converting metadata to JSON format: %v ...", err)
return fmt.Errorf("problem converting metadata to JSON format: %s", err)
}
b := bytes.NewBuffer(mdJSON)
uri := conf.Host + ":" + strconv.Itoa(int(conf.Port)) + "/api/deposit/depositions/" + id + "?access_token=" + conf.Token
log.Printf("Post request to: %s", uri)
client := &http.Client{}
req, err := http.NewRequest("POST", uri, b)
if err != nil {
log.Printf("Problem creating POST request: %v ...", err)
return fmt.Errorf("problem creating POST request: %s", err)
}
req.Header.Add("Content-Type", `application/json`)
// func PutMetadata
resp, err := client.Do(req)
if err != nil {
log.Printf("Problem sending POST request: %v ...", err)
return fmt.Errorf("problem sending POST request: %s", err)
}
defer resp.Body.Close()
content, err := ioutil.ReadAll(resp.Body)
if err != nil {
log.Printf("Problem reading POST response: %v ...", err)
return fmt.Errorf("problem reading POST response: %s", err)
}
if strconv.Itoa(resp.StatusCode)[:1] != "2" {
log.Printf("Problem reported by zenodo: %d %v. %80s ...", resp.StatusCode, err, content)
return fmt.Errorf("problem reported by zenodo: %v. %s", err, content)
}
log.Printf("Response code %d, content: %80s", resp.StatusCode, content)
return nil
}
}
// func Publish
......
This source diff could not be displayed because it is too large. You can view the blob instead.
Markdown is supported
0%