Commit 88273d3e authored by Marcel Hellkamp's avatar Marcel Hellkamp
Browse files

UI concept idea: High level sync functionality + low level operations

parent e4ac9555
......@@ -11,6 +11,46 @@ CDSTAR 2 and [pycdstar3](https://gitlab.gwdg.de/cdstar/pycdstar3) for CDSTAR 3
and newer.
## Command-Line interface
`cdstar3` is a command-line toolbox to upload, download or manage data in a CDSTAR repository.
Please note that `cdstar3` was designed to be used by humans, not scripts. The output is mostly human-friendly and may
change between releases. If you want to automate CDSTAR, consider implementing your tools using the `pycdstar3` client library, or directly against the stable CDSTAR REST API.
### Configuration
The `cdstar3` client needs to know which server to connect to and which vault to use per default. This information (and more) is defined in a file named `cdstar.conf`. If no such file is specified via the `-c` parameter, `cdstar3` will look for a `cdstar.conf` in the current working directory, and then in any of its parent directories. As a last resort, certain system-dependent user folders are searched (e.g. `~/.config/cdstar3/` on linux). You can create a configuration file with `cdstar3 init`.
### Usage
This is an (incomplete) list of commands and their most important parameters. For a complete list, run `cdstar3 -h` and for details, see `cdstar3 COMMAND -h`.
* **`init`**: Ask for server address, vault, credentials and other config options and create a `cdstar.conf` file in the current directory.
High-level commands work with local archive directories (one per archive). When creating a new archive or recovering an existing archive for the first time, `cdstar3` will create a hidden `.cdstar` folder within the target directory and remember the exact location (server, vault and id) of the remote archive. Do NOT delete this folder, or the correlation between your local copy and the remote archive is lost.
* **`archive DIR`**: Create a new remote archive from a local directory.
* **`recover ARCHIVE DIR`** download an existing remote archive to a local directory.
* **`sync --up/--down`**: Synchronize a local archive directory with the corresponding remote archive by upload- or downloading missing files. This command is *save* by default: Existing files are not overwritten and no files are removed.
* `--update` overwrite outdated files at the target location (comparing last modified time).
* `--force` overwrite all files that do not match, even if the target file is newer.
* `--delete` remove files at the target location that are not present at the source.
* `--progress` show a fancy progress bar.
* `--dry-run` only print what would have changed, but do not actually apply any changes.
* `-- [files]` only sync these files or folders.
Low-level commands do not require a local archive directory and operate directly on remote archives or vaults. If called from within a local archive directory however, the remote archive can be referred to as `origin`. For example, `cdstar3 ls origin` will list all files in the remote archive.
* **`search QUERY`** Search a vault.
* **`scroll [START]`** List all IDs in a vault, starting with the given id.
* **`info [ARCHIVE]`**: Print information about a remote archive.
* **`ls ARCHIVE`** List all files in an remote archive.
* **`get ARCHIVE NAME [FILE]`** Download a single file from a remote archive.
* **`put ARCHIVE FILE [NAME]`** Upload a single file to a remote archive.
* **`meta get/set ARCHIVE ATTR [VALUES]`** Read or set the value of a meta attribute.
* **`acl ARCHIVE SUBJECT [PERMISSIONS]`** Set permissions for a specific subject.
## License
......
......@@ -4,13 +4,13 @@ import importlib
import os
import re
import sys
from typing import Tuple, Dict
from .context import CliContext
from ..cdstar import CDStar
__ALL__ = ["main", "register_subcommand", "printer"]
CONFIG_NAMES = ["cdstar.conf"]
#: Commands to load automatically. The module pycdstar.cli.NAME must have register() defined.
BUILDIN_COMMANDS = {"init", "put"}
......@@ -100,7 +100,7 @@ printer = Printer(level=0, file=sys.stderr)
def main(args=None):
# Load and register all built-in commands
for name in BUILDIN_COMMANDS:
importlib.import_module("."+name, __name__).register()
importlib.import_module(".commands."+name, __name__).register()
# Parse command line arguments (may fail)
opts = parser.parse_args(args)
......@@ -123,22 +123,8 @@ def main(args=None):
cmdmain = __subcommands[cmd]
try:
if cmd == "init":
return cmdmain(opts) or 0
config = load_config(opts.config) if opts.config else find_config(os.getcwd())
server = config["DEFAULT"]["server"]
vault = config["DEFAULT"]["vault"]
auth = config["DEFAULT"]["auth"]
if auth.startswith("basic:"):
auth = tuple(auth[6:].split(':', 1))
else:
raise ValueError("Unknown authentication setting format (not printed, see config file)")
client = CDStar(server, auth=auth)
return cmdmain(client, vault, opts) or 0
ctx = CliContext(workdir=".")
return cmdmain(ctx, opts) or 0
except KeyboardInterrupt:
printer("Exiting...")
return 0
......@@ -147,77 +133,10 @@ def main(args=None):
return e.return_code
except Exception as e:
printer.fatal("Uncaught exception. Exiting...")
raise
return 1
def load_config(fname):
printer.vv("Loading config from: {}", fname)
config = configparser.ConfigParser()
config.read(fname)
return config
def find_config(start_dir):
current_dir = os.path.abspath(start_dir)
while os.path.isdir(current_dir):
printer.vvv("Searching for config file in: {}", current_dir)
for name in CONFIG_NAMES:
fname = os.path.join(current_dir, name)
if os.path.exists(fname):
return load_config(fname)
parent = os.path.abspath(os.path.join(current_dir, os.pardir))
if parent == current_dir:
break
current_dir = parent
raise CliError("Could not find '{}' in '{}' or any parent directory.", CONFIG_NAMES[0], start_dir)
def hbytes(n):
for unit in ('B', 'KiB', 'MiB', 'GiB', 'TiB', 'PiB', 'EiB', 'ZiB', 'YiB'):
if abs(n) < 1024.0:
return "{:.1f} {}".format(n, unit)
n /= 1024.0
return "alot"
def compile_glob(pattern):
parts = re.split(r'(\*\*|\*|\?)', pattern)
res = ["^" if pattern.startswith("/") else ".*"]
for i, part in enumerate(parts):
if i % 2 == 0:
res.append(re.escape(part))
elif part == '*':
res.append(r'[^/]+')
elif part == '**':
res.append(r'.+')
elif part == '?':
res.append(r'[^/]')
return re.compile(''.join(res) + "$")
class FileProgress:
def __init__(self, fp, desc=None):
from requests.utils import super_len
self.desc = desc
self.fp = fp
self.len = super_len(fp)
def __iter__(self):
from tqdm import tqdm
with tqdm(total=self.len, unit='b', unit_scale=True, unit_divisor=1024, dynamic_ncols=True,
leave=False) as pbar:
read = self.fp.read
update = pbar.update
while True:
chunk = read(1024 * 64)
update(len(chunk))
if not chunk:
break
yield chunk
pbar.close()
class CliError(Exception):
......@@ -226,5 +145,5 @@ class CliError(Exception):
The message will still be printed. """
def __init__(self, *args, status=1):
super(self, Exception).__init__(*args)
super().__init__(*args)
self.return_code = status
import os
import re
def walk_up(start):
""" Yield the input and all its parent directories. """
current_dir = os.path.abspath(start)
while os.path.isdir(current_dir):
yield current_dir
parent = os.path.abspath(os.path.join(current_dir, os.pardir))
if parent == current_dir:
break
current_dir = parent
def hbytes(n):
for unit in ('B', 'KiB', 'MiB', 'GiB', 'TiB', 'PiB', 'EiB', 'ZiB', 'YiB'):
if abs(n) < 1024.0:
return "{:.1f} {}".format(n, unit)
n /= 1024.0
return "alot"
def compile_glob(pattern):
parts = re.split(r'(\*\*|\*|\?)', pattern)
res = ["^" if pattern.startswith("/") else ".*"]
for i, part in enumerate(parts):
if i % 2 == 0:
res.append(re.escape(part))
elif part == '*':
res.append(r'[^/]+')
elif part == '**':
res.append(r'.+')
elif part == '?':
res.append(r'[^/]')
return re.compile(''.join(res) + "$")
class FileProgress:
def __init__(self, fp, chunksize=1024*8, **baropts):
from requests.utils import super_len
self.opts = baropts
self.fp = fp
self.len = super_len(fp)
self.chunksize = chunksize
def __iter__(self):
from tqdm import tqdm
with tqdm(total=self.len, unit='b', unit_scale=True, unit_divisor=1024, dynamic_ncols=True, **self.opts) as pbar:
read = self.fp.read
update = pbar.update
while True:
chunk = read(self.chunksize)
if not chunk:
break
update(len(chunk))
yield chunk
pbar.close()
......@@ -12,7 +12,7 @@ import configparser
import os
import re
from . import CONFIG_NAMES, register_subcommand, printer
from cli import CONFIG_NAMES, register_subcommand, printer
def register():
......@@ -24,7 +24,7 @@ def register():
grp.add_argument("--auth", help="Login credentials as a username:password string")
def command(cdstar, vault, args):
def command(ctx, args):
def ask(q, default=None, rx=None):
while True:
val = input("{}? [{}] ".format(q, default) if default else "{}? ".format(q)).strip()
......@@ -58,9 +58,9 @@ def command(cdstar, vault, args):
config["DEFAULT"]["auth"] = ':'.join(auth)
with open(target, 'w') as fp:
fp.write("# cdstar-cli config. See https://cdstar.gwdg.de/\n".format(sys.argv[0]))
fp.write("# cdstar-cli config. See https://cdstar.gwdg.de/\n")
config.write(fp)
printer("Okay!")
printer("")
printer("Config file written to:", target)
printer("Config file written to: {}", target)
......@@ -3,7 +3,7 @@ Upload one or more files to an existing archive.
"""
import os
from . import register_subcommand, compile_glob, FileProgress, hbytes, printer, CliError
from cli import register_subcommand, compile_glob, FileProgress, hbytes, printer, CliError
def register():
......@@ -17,7 +17,8 @@ def register():
parser.add_argument("-x", "--exclude", metavar="GLOB", action="append", help="Exclude files by glob pattern")
parser.add_argument("-i", "--include", metavar="GLOB", action="append",
help="Include files by glob pattern (default: all)")
parser.add_argument("-p", "--progress", action="store_true", help="Show progress")
parser.add_argument("-p", "--progress", action="store_true",
help="Show progress bar for large files or slow uploads")
parser.add_argument("archive", help="Archive ID, or 'new' to create a new archive")
parser.add_argument("file", nargs='+', help="File(s) (or directories) to upload")
......@@ -41,8 +42,9 @@ def findfiles(flist, recursive=False, include_hidden=False):
raise CliError("Not a file: " + entry)
def command(cdstar, vault, args):
def command(ctx, args):
archive = args.archive
vault = ctx.default_vault
inc = [compile_glob(rule).match for rule in args.include or []]
exc = [compile_glob(rule).match for rule in args.exclude or []]
progress = args.progress
......@@ -74,18 +76,31 @@ def command(cdstar, vault, args):
len(uploads), hbytes(total), "new archive" if archive == 'new' else "archive: " + vault + "/" + archive)
return
with cdstar.begin(autocommit=True) as ctx:
with ctx.client.begin(autocommit=True) as ctx:
if archive == 'new':
archive = cdstar.create_archive(vault)['id']
archive = ctx.client.create_archive(vault)['id']
printer("Uploading {} files ({}) to new archive: {}/{}", len(uploads), hbytes(total), vault, archive)
else:
printer("Uploading {} files ({}) to archive: {}/{}", len(uploads), hbytes(total), vault, archive)
pbar = None
if progress and not printer.quiet:
from tqdm import tqdm
total = sum(stat.st_size for (file, stat) in uploads.values())
pbar = tqdm(total=total, unit='b', unit_scale=True, unit_divisor=1024, dynamic_ncols=True,
file=printer.file)
for i, target in enumerate(sorted(uploads)):
file, stat = uploads[target]
with open(file, 'rb') as fp:
printer("[{}/{}] {}", i + 1, len(uploads), target)
if progress and not printer.quiet:
fp = FileProgress(fp, printer.file)
cdstar.put_file(vault, archive, target, fp)
line = "[{}/{}] {}".format(i + 1, len(uploads), target[1:])
if pbar:
pbar.write(line)
read = fp.read
chunks = iter(lambda: read(1024*8), b'')
chunks = (chunk for chunk in chunks if not pbar.update(len(chunk)))
fp = chunks
else:
printer(line)
ctx.client.put_file(vault, archive, target, fp)
printer("Done")
import os
import urllib.parse
from . import printer, CliError
from ._utils import walk_up
from ..cdstar import CDStar
METADIR_NAME = '.cdstar'
CONFIG_NAMES = ["cdstar.conf", ".cdstar.conf"]
class CliContext:
""" Search for config files and archive directory in or above the current working directory. """
def __init__(self, workdir="."):
self.workdir = os.path.abspath(workdir)
self.archive_root = None
self.config = None
self._client = None
home = os.path.abspath(os.path.expanduser("~"))
for path in walk_up(self.workdir):
if not self.archive_root:
if os.path.exists(os.path.join(path, METADIR_NAME)):
printer.vv("Found archive directory: {}", path)
self.archive_root = path
if not self.config:
for name in CONFIG_NAMES:
cfile = os.path.join(path, name)
if os.path.exists(cfile):
printer.vv("Found config file: {}", cfile)
self.config = Config(cfile)
break
if path == home:
return
@property
def default_vault(self):
if not self.config or 'vault' not in self.config:
raise CliError("No default vault configured")
return self.config['vault']
@property
def client(self):
if self._client:
return self._client
if not self.config:
raise CliError("Unable to connect: No config found (see `init` command)")
server = self.config["server"]
url = urllib.parse.urlparse(server)
if url.username:
auth = (url.username, url.password or self._ask_pass())
self._client = CDStar(server, auth=auth)
else:
self._client = CDStar(server)
return self._client
def _ask_pass(self):
raise RuntimeError("Asking for passwort not implemented yet")
class ArchiveDir:
""" A folder containing a `.cdstar` archive metadata directory. """
def __init__(self, path):
if not os.path.isdir(path):
raise IOError("Not a directory: " + path)
if not os.path.exists(os.path.join(path, METADIR_NAME)):
raise IOError("Already initialized: " + path)
self.root = path
self.metadir = os.path.join(path, METADIR_NAME)
class Config:
def __init__(self, path):
import configparser
self.path = os.path.abspath(path)
cfg = configparser.ConfigParser()
cfg.read(path)
self.data = dict(cfg["DEFAULT"])
def __iter__(self):
return iter(self.data)
def __getitem__(self, item):
try:
return self.data[item]
except KeyError as e:
raise KeyError("Missing config field: " + item)
def __contains__(self, item):
return item in self.data
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment