Dear Gitlab users, due to maintenance reasons, Gitlab will not be available on Thursday 30.09.2021 from 5:00 pm to approximately 5:30 pm.

Commit 864e0126 authored by Andreas Wagner's avatar Andreas Wagner
Browse files

Start working on github putfile method.

parent b95ecc17
This diff is collapsed.
......@@ -22,13 +22,11 @@ func main() {
Cerr := conf.Configure(&Config)
if Cerr != nil {
panic(fmt.Errorf("Error during config phase_ %+v", Cerr))
return
}
log, Lerr := logger.ConfigureLogging(&Config.Log)
if Lerr != nil {
panic(fmt.Errorf("Error during log setup phase: %+v", Lerr))
return
}
log.Printf("Starting tei2zenodo daemon")
......
......@@ -9,17 +9,18 @@
"level": "Info"
},
"Zenodo": {
"prefix": "10.5072/zenodo.",
"token": "aBcDeFgHiJkLmNoPqRsTuVwXyZ",
"host": "https://sandbox.zenodo.org",
"token": "aBcDeFgHiJkLmNoPqRsTuVwXyZ"
"prefix": "10.5072/zenodo."
},
"Git": {
"host": "https://api.github.com",
"token": "aBcDeFgHiJkLmNoPqRsTuVwXyZ",
"secret": "ThIs_iS_NoT_ReAlLy_a_sMaRt_sEcReT!!",
"user": "octocat",
"repo": "octocat/hello-world",
"commit_phrase": "",
"branch": "public",
"hook_user": "foobar",
"commit_phrase": "(push to zenodo)",
"commit_dontpublish_phrase": "test"
},
"metadata": {
......
......@@ -2,6 +2,7 @@ package conf
import (
"fmt"
"os"
"strings"
"github.com/spf13/viper"
......@@ -31,8 +32,9 @@ func Configure(Config *tei2zenodo.Config) error {
"token": "jLsTkUOMDU2fnGdGivGbB9TnMkPcADhIzEKHqqoVzMRsdYEC0Sqqfz72SPpt"})
viper.SetDefault("Git", map[string]string{"host": "https://api.github.com",
"token": "",
"user": "octocat",
"repo": "octocat/hello-world",
"branch": "master",
"hook_user": "foobar",
"commit_phrase": "",
"commit_dontpublish_phrase": "test"})
......@@ -44,17 +46,15 @@ func Configure(Config *tei2zenodo.Config) error {
viper.SetConfigName("config")
viper.SetConfigType("json")
viper.AddConfigPath("./")
viper.AddConfigPath("./configs/")
viper.AddConfigPath("$HOME/.t2z")
viper.AddConfigPath(os.Getenv("XDG_CONFIG_HOME") + "/t2z")
viper.AddConfigPath(os.Getenv("HOME") + "/.t2z")
viper.AddConfigPath("/etc/t2z/")
if err := viper.ReadInConfig(); err != nil {
if _, ok := err.(viper.ConfigFileNotFoundError); ok {
panic(fmt.Errorf("Config file not found"))
return err
} else {
panic(fmt.Errorf("Cannot read config in %s: %+v", viper.ConfigFileUsed(), err))
return err
}
panic(fmt.Errorf("Cannot read config in %s: %+v", viper.ConfigFileUsed(), err))
}
/*
......@@ -67,7 +67,6 @@ func Configure(Config *tei2zenodo.Config) error {
err := viper.Unmarshal(Config)
if err != nil {
panic(fmt.Errorf("Cannot parse config: %+v", err))
return err
}
return nil
......
......@@ -14,7 +14,10 @@ import (
"strings"
"time"
"github.com/gin-gonic/gin"
"github.com/google/go-github/github"
log "github.com/sirupsen/logrus"
"golang.org/x/oauth2"
"gitlab.gwdg.de/rg-mpg-de/tei2zenodo"
)
......@@ -271,7 +274,7 @@ type Repo struct {
ContributorsURL string `json:"contributors_url"`
SubscribersURL string `json:"subscribers_url"`
SubscriptionURL string `json:"subscription_url"`
CommitsURL string `json:"commits_url"`
CommitsURL string `json:"commits_url"` // we use this
GitCommitsURL string `json:"git_commits_url"`
CommentsURL string `json:"comments_url"`
IssueCommentsURL string `json:"issue_comments_url"`
......@@ -297,11 +300,14 @@ type Repo struct {
// FileInfo stores a file URL, modification type and modification timestamp
// (to be used as value in a map keyed to the filenames)
type FileInfo struct {
URL string
BlobURL string
Change string
SHA string
CommitSHA string
Date time.Time
DoPublish bool
Filename string
ObjectSHA string
RawURL string
}
// ====== Functions =======
......@@ -340,24 +346,24 @@ func ProcessHook(hookType string, r io.ReadSeeker, conf *tei2zenodo.Config) (boo
}
commitsURL := payload.Repository.CommitsURL
files := make(map[string]FileInfo)
// Get all commits and their associated files
log.Printf("--- Get all commits and their associated files ---")
files := make(map[string]FileInfo)
for _, c := range payload.Commits {
sha := c.URL[strings.LastIndex(c.URL, "/")+1:]
commitSHA := c.URL[strings.LastIndex(c.URL, "/")+1:]
if conf.Verbose {
log.Printf(" commit %s ...", sha)
log.Printf(" commit %s ...", commitSHA)
}
if !(strings.Contains(c.Message, conf.Git.Phrase)) && conf.Git.Phrase != "" {
if conf.Verbose {
log.Printf("Required phrase not contained in commit message for commit %s", sha)
log.Printf("Required phrase not contained in commit message for commit %s", commitSHA)
}
continue
}
f, err := retrieveFiles(commitsURL, sha, conf)
f, err := retrieveFiles(commitsURL, commitSHA, conf)
if err != nil {
log.Printf("Error retrieving files from commit %s: %+v", sha, err)
return false, nil, tei2zenodo.NewError("errInternal", fmt.Sprintf("error retrieving files from commit %s: %s", sha, err), 500, err)
log.Printf("Error retrieving files from commit %s: %+v", commitSHA, err)
return false, nil, tei2zenodo.NewError("errInternal", fmt.Sprintf("error retrieving files from commit %s: %s", commitSHA, err), 500, err)
}
for k, v := range f {
......@@ -395,11 +401,11 @@ func ProcessHook(hookType string, r io.ReadSeeker, conf *tei2zenodo.Config) (boo
}
}
func retrieveFiles(commitsURL string, sha string, conf *tei2zenodo.Config) (map[string]FileInfo, error) {
func retrieveFiles(commitsURL string, commitSHA string, conf *tei2zenodo.Config) (map[string]FileInfo, error) {
files := make(map[string]FileInfo)
// Compile GET request to retrieve commit information
targetURI := strings.Replace(commitsURL, "{/sha}", "/"+sha, -1)
targetURI := strings.Replace(commitsURL, "{/sha}", "/"+commitSHA, -1)
req, err := http.NewRequest("GET", targetURI, nil)
req.Header.Add("Authorization", "token "+conf.Git.Token)
req.Header.Add("Accept", "application/vnd.github.v3+json")
......@@ -440,7 +446,7 @@ func retrieveFiles(commitsURL string, sha string, conf *tei2zenodo.Config) (map[
var doPublish bool
if !(strings.Contains(commit.Commit.Message, conf.Git.DontPublishPhrase)) && conf.Git.DontPublishPhrase != "" {
log.Printf("!!! Don't-publish-Phrase '%s' contained in commit message for commit %s,\n", conf.Git.DontPublishPhrase, sha)
log.Printf("!!! Don't-publish-Phrase '%s' contained in commit message for commit %s,\n", conf.Git.DontPublishPhrase, commitSHA)
doPublish = false
} else {
doPublish = true
......@@ -453,8 +459,9 @@ func retrieveFiles(commitsURL string, sha string, conf *tei2zenodo.Config) (map[
counter := 0
for _, f := range commit.Files {
filename := f.Filename
url := f.RawURL
sha := f.SHA
rawURL := f.RawURL
blobURL := f.BlobURL
objectSHA := f.SHA
modType := f.Status
if modType != "deleted" && modType != "renamed" {
counter++
......@@ -466,7 +473,7 @@ func retrieveFiles(commitsURL string, sha string, conf *tei2zenodo.Config) (map[
if !doPublish {
log.Printf("!!! not going to publish zenodo deposit for %s !!!", filename)
}
files[filename] = FileInfo{URL: url, Change: modType, Date: commitDate, SHA: sha, DoPublish: doPublish}
files[filename] = FileInfo{Filename: filename, RawURL: rawURL, BlobURL: blobURL, Change: modType, CommitSHA: commitSHA, Date: commitDate, ObjectSHA: objectSHA, DoPublish: doPublish}
} else {
if conf.Verbose {
log.Printf(" ignore %s file %s: %s", modType, filename, url)
......@@ -476,13 +483,91 @@ func retrieveFiles(commitsURL string, sha string, conf *tei2zenodo.Config) (map[
return files, nil
}
// PutFile uploads a file to the github repository
// DownloadFile takes a Deposit downloads the file from githubRawURL
// it returns a ReadSeeker and an error value
func DownloadFile(myDeposit *tei2zenodo.Deposit, conf *tei2zenodo.Config) (r io.ReadSeeker, err *tei2zenodo.Error) {
// Compile GET request
targetURI := myDeposit.GithubRawURL
req, RQErr := http.NewRequest("GET", targetURI, nil)
if RQErr != nil {
log.Printf("Problem creating GET request: %v ...", RQErr)
return nil, tei2zenodo.NewError("errInternal", fmt.Sprintf("error creating GET request: %s", RQErr.Error()), 500, RQErr)
}
req.Header.Add("Authorization", "Bearer "+conf.Git.Token)
req.Header.Add("User-Agent", "tei2zenodo (https://gitlab.gwdg.de/rg-mpg-de/tei2zenodo/)")
// Send GET request
client := &http.Client{}
resp, RSPErr := client.Do(req)
if RSPErr != nil {
log.Printf("Problem sending GET request: %v ...", RSPErr)
return nil, tei2zenodo.NewError("errNetComm", fmt.Sprintf("error sending GET request: %s", RSPErr.Error()), 500, RSPErr)
}
defer resp.Body.Close()
// Read file from Request body
var r io.ReadSeeker
buf := new(bytes.Buffer)
buf.ReadFrom(resp.Body)
file := buf.String()
r = strings.NewReadSeeker(file)
return r, nil
}
// PutFile uploads a file to the github repository and commits
// it returns the commit sha and an error value
func PutFile(myDeposit *tei2zenodo.Deposit, c *tei2zenodo.Config) (string, error) {
func PutFile(myDeposit *tei2zenodo.Deposit, conf *tei2zenodo.Config, c *gin.Context) (string, error) {
// authenticate with Personal Access Token/Oauth2
ts := oauth2.StaticTokenSource(
&oauth2.Token{AccessToken: githubConf.Token},
)
tc := oauth2.NewClient(c, ts)
client := github.NewClient(tc)
// 1. remember old commit SHA (myDeposit.CommitSHA)
// 2. get old tree
// 3. create new blob object(s) (get new SHA(s))
// 4. create and post new tree object (get new SHA)
// 5. create and post new commit object (use old commit SHA as parent and new tree SHA; get new SHA)
// 6. update heads/{branch} to point to new commit SHA
githubConf := conf.Git
ownerName := strings.Split(githubConf.Repo, "/")[0]
repoName := strings.Split(githubConf.Repo, "/")[1]
owner, _, err := client.Users.Get(c, ownerName)
repo, _, err := client.Repositories.Get(c, ownerName, repoName)
conf := c.Git
filename := myDeposit.Filename
sha := myDeposit.FileGithubSHA
oldCommitSHA := myDeposit.CommitSHA
oldCommit, _, err := client.Git.GetCommit(c, ownerName, repoName, oldCommitSHA)
baseTree := oldCommit.GetTree()
baseTreeSHA := baseTree.SHA
entries := []*github.TreeEntry{}
newTree, _, err := client.Git.CreateTree(c, ownerName, repoName, *baseTreeSHA, entries)
// Create the commit using the new tree.
commitAuthor := "tei2zenodo service"
commitAuthorEMail := "bla@foo.com"
commitMessage := "Zenodo DOI updated"
date := time.Now()
author := &github.CommitAuthor{Date: &date, Name: &commitAuthor, Email: &commitAuthorEMail}
commit := &github.Commit{Author: author, Message: &commitMessage, Tree: newTree, Parents: []*github.Commit{oldCommit}}
newCommit, _, err := client.Git.CreateCommit(c, ownerName, repoName, commit)
if err != nil {
return err
}
// Attach the commit to the master branch.
ref.Object.SHA = newCommit.SHA
_, _, err = client.Git.UpdateRef(c, ownerName, repoName, ref, false)
return err
log.Printf("--- Upload new version to github ---")
// Read file from myDeposit
......@@ -493,8 +578,8 @@ func PutFile(myDeposit *tei2zenodo.Deposit, c *tei2zenodo.Config) (string, error
// For testing: uri := "https://postman-echo.com/post"
// https://api.github.com/repos/:owner/:repo/contents/:path
uri := conf.Host + "/repos/" + conf.Repo + "/contents/" + filename // ?access_token=" + conf.Token
if c.Verbose {
uri := githubConf.Host + "/repos/" + githubConf.Repo + "/contents/" + filename // ?access_token=" + conf.Token
if conf.Verbose {
log.Printf(" Put request to: %s", uri)
log.Printf(" Upload filename: %s", filename)
log.Printf(" Upload file: %s ...", myDeposit.FileContent[:80])
......@@ -523,7 +608,7 @@ func PutFile(myDeposit *tei2zenodo.Deposit, c *tei2zenodo.Config) (string, error
log.Printf("Problem creating PUT request body: %v ...", err)
return "", fmt.Errorf("problem creating PUT request body: %s", err)
}
shaID.Write([]byte(sha))
shaID.Write([]byte(oldSHA))
uploadContent, err := writer.CreateFormField("content")
if err != nil {
......@@ -552,7 +637,7 @@ func PutFile(myDeposit *tei2zenodo.Deposit, c *tei2zenodo.Config) (string, error
return "", fmt.Errorf("problem creating PUT request: %s", err)
}
// req.Header.Add(`Content-Type`, writer.FormDataContentType())
req.Header.Add("Authorization", "token "+conf.Token)
req.Header.Add("Authorization", "token "+githubConf.Token)
req.Header.Add("Accept", "application/vnd.github.v3+json")
req.Header.Add("User-Agent", "tei2zenodo (https://gitlab.gwdg.de/rg-mpg-de/tei2zenodo/)")
......@@ -585,7 +670,7 @@ func PutFile(myDeposit *tei2zenodo.Deposit, c *tei2zenodo.Config) (string, error
}
// log.Printf("Response code %d, content: %s", resp.StatusCode, content)
if c.Verbose {
if conf.Verbose {
log.Printf(" Success (code %d (expected 200), %s Bytes). Commit SHA: %s", resp.StatusCode, strconv.Itoa(int(parsedContent.Content.Size)), parsedContent.Commit.SHA)
} else {
log.Printf(" Success.")
......
......@@ -27,7 +27,7 @@ import (
func SetupRoutes(conf tei2zenodo.Config) *gin.Engine {
// Switch gin to release mode
gin.SetMode(gin.ReleaseMode)
// gin.SetMode(gin.ReleaseMode)
// Create a gin router with logrus router and stock recovery
router := gin.New()
......@@ -162,28 +162,48 @@ func SetupRoutes(conf tei2zenodo.Config) *gin.Engine {
for f := range files {
log.Printf("\n")
log.Printf("Processing file '%s'", f)
var myDeposit tei2zenodo.Deposit
// myDeposit.Filename = strings.Replace(f, "/", "_", -1)
myDeposit.Filename = f
myDeposit.FileGithubSHA = files[f].SHA
myDeposit.CommitSHA = files[f].CommitSHA
myDeposit.DoPublish = files[f].DoPublish
myDeposit.FileGHURI = files[f].URL
PDErr := zenodo.ProcessDownloadFile(&myDeposit, &conf)
if PDErr != nil {
switch PDErr.Typ {
myDeposit.Filename = f
myDeposit.GithubBlobURL = files[f].BlobURL
myDeposit.GithubObjSHA = files[f].ObjectSHA
myDeposit.GithubRawURL = files[f].RawURL
r, GDLErr := github.DownloadFile(&myDeposit, &conf)
if GDLErr != nil {
log.Printf("Problem downloading file %s (%s): %v ...", myDeposit.Filename, myDeposit.GithubObjSHA, GDLErr)
AbortMsg(500, tei2zenodo.NewError("errNetComm", fmt.Sprintf("error downloading file %s (%s): %s", myDeposit.Filename, myDeposit.GithubObjSHA, GDLErr.Error()), 500, GDLErr), c)
return
}
ZPFErr := zenodo.ProcessFile(r, &myDeposit, &conf)
if ZPFErr != nil {
switch ZPFErr.Typ {
case "errNoTEIXML":
{
log.Printf(" No TEI XML file, skipping...")
continue
log.Printf("Problem with file %s (%s): No TEI file.", myDeposit.Filename, myDeposit.GithubObjSHA)
AbortMsg(500, tei2zenodo.NewError("errZProcessing", fmt.Sprintf("problem with file %s (%s): No TEI file.", myDeposit.Filename, myDeposit.GithubObjSHA), 500, ZPFErr), c)
return
}
default:
{
log.Printf("Problem processing file %s: %+v", myDeposit.Filename, PDErr)
AbortMsg(500, tei2zenodo.NewError("errZProcessing", fmt.Sprintf("problem processing file %s", myDeposit.Filename), 500, PDErr), c)
log.Printf("Problem processing file %s: %v ...", myDeposit.Filename, ZPFErr)
AbortMsg(500, tei2zenodo.NewError("errZProcessing", fmt.Sprintf("problem processing file %s (%s): %s", myDeposit.Filename, myDeposit.GithubObjSHA, ZPFErr.Error()), 500, ZPFErr), c)
return
}
}
}
_, GPFErr := github.PutFile(&myDeposit, &conf, c)
if GPFErr != nil {
log.Printf("Error putting file to github: %v", GPFErr)
AbortMsg(500, tei2zenodo.NewError("errPutFile", fmt.Sprintf("problem putting file to github: %v", f), 500, GPFErr), c)
return
}
/*
// The upload to github requires more than just a token:
// you need to authenticate an app to act in lieu of the user in github...
......
......@@ -48,54 +48,8 @@ type ZPostData struct {
// ====== Functions ======
// ProcessDownloadFile takes a URL, downloads the file and continues processing
func ProcessDownloadFile(myDeposit *tei2zenodo.Deposit, conf *tei2zenodo.Config) *tei2zenodo.Error {
// Compile GET request
targetURI := myDeposit.FileGHURI
req, RQErr := http.NewRequest("GET", targetURI, nil)
if RQErr != nil {
log.Printf("Problem creating GET request: %v ...", RQErr)
return tei2zenodo.NewError("errInternal", fmt.Sprintf("error creating GET request: %s", RQErr.Error()), 500, RQErr)
}
req.Header.Add("Authorization", "Bearer "+conf.Git.Token)
req.Header.Add("User-Agent", "tei2zenodo (https://gitlab.gwdg.de/rg-mpg-de/tei2zenodo/)")
// Send GET request
client := &http.Client{}
resp, RSPErr := client.Do(req)
if RSPErr != nil {
log.Printf("Problem sending GET request: %v ...", RSPErr)
return tei2zenodo.NewError("errNetComm", fmt.Sprintf("error sending GET request: %s", RSPErr.Error()), 500, RSPErr)
}
defer resp.Body.Close()
// Read file from Request body
var r io.ReadSeeker
buf := new(bytes.Buffer)
buf.ReadFrom(resp.Body)
file := buf.String()
r = strings.NewReader(file)
PFErr := ProcessFile(r, myDeposit, conf)
if PFErr != nil {
switch PFErr.Typ {
case "errNoTEIXML":
{
return PFErr
}
default:
{
log.Printf("Problem processing file %s: %v ...", myDeposit.Filename, PFErr)
return tei2zenodo.NewError("errZProcessing", fmt.Sprintf("error processing file %s: %s", myDeposit.Filename, PFErr.Error()), 500, PFErr)
}
}
}
return nil
}
// ProcessFile takes a file and processes parsing and zenodo upload
func ProcessFile(r io.ReadSeeker, myDeposit *tei2zenodo.Deposit, conf *tei2zenodo.Config) *tei2zenodo.Error {
func ProcessFile(*r io.ReadSeeker, myDeposit *tei2zenodo.Deposit, conf *tei2zenodo.Config) *tei2zenodo.Error {
doPublish := myDeposit.DoPublish
......
......@@ -33,8 +33,9 @@ type GitConfig struct {
Host string
Token string
Secret string
User string
Repo string
Branch string
HookUser string `json:"hook_user"`
Phrase string `json:"commit_phrase"`
DontPublishPhrase string `json:"commit_dontpublish_phrase"`
}
......@@ -53,18 +54,20 @@ type metadataField struct {
// Deposit stores a zenodo Deposit, part of it is returned by this service
type Deposit struct {
CommitSHA string
ConceptDOI string `json:"conceptdoi"`
ConceptRecID string `json:"conceptrecid"`
Created string
DepositDOI string `json:"doi"`
DOIURL string `json:"doi_url"`
DoPublish bool
FileContent string
Filename string
FileGithubSHA string
Files []ZFile
Filesize int64
FileZURI string
FileGHURI string
GithubBlobURL string
GithubObjSHA string
GithubRawURL string
ID int64
Links ZLinks
Metadata ZMetadata `json:"metadata"`
......@@ -75,7 +78,7 @@ type Deposit struct {
State string
Submitted bool
Title string
DoPublish bool
ZenodoURL string
}
// ZFile stores a zenodo file response
......
......@@ -2,9 +2,6 @@
type="application/xml"
schematypens="http://relaxng.org/ns/structure/1.0"?>
<TEI xmlns="http://www.tei-c.org/ns/1.0" xmlns:xi="http://www.w3.org/2001/XInclude" xml:id="W0004">
<?svsal htmlFragmentationDepth="4" ?>
<?svsal htmlFragmentationDepth="4" ?>
<?svsal htmlFragmentationDepth="4" ?>
<?svsal htmlFragmentationDepth="4" ?>
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment