Commit 4e3d6ce4 authored by Denis Zalevskiy's avatar Denis Zalevskiy

git-remote-vault copies blobs and tags

Signed-off-by: Denis Zalevskiy's avatarDenis Zalevskiy <denis.zalevskiy@jolla.com>
parent c6c23bf4
......@@ -91,6 +91,7 @@ rm -rf $RPM_BUILD_ROOT
%{_bindir}/vault
%{_bindir}/vault-sync
%{_bindir}/vault-resolve
%{_bindir}/git-remote-vault
%dir %{tools_dir}
%{tools_dir}/*
%if 0%{?_with_usersession:1}
......
......@@ -8,11 +8,20 @@ install(
DESTINATION ${TOOLS_DIR}
)
install(
PROGRAMS
git-remote-vault
DESTINATION bin
)
install(
FILES
git-vault-rebase-generate.awk git-vault-rebase-prepare.awk
git-vault-rebase-generate.awk
git-vault-rebase-prepare.awk
git-vault-split-refs.awk
vault-misc
vault-unit-common
git-vault-helpers-remote
DESTINATION
${TOOLS_DIR}
)
#!/bin/bash
function error {
echo "`basename $0`: Error: ${@:1}" 1>&2;
src=$(dirname $0)
src=$(cd $src;pwd)
util_file=
if [ "x$VAULT_LIB_DIR" != "x" ]; then
util_file=$VAULT_LIB_DIR/git-vault-helpers-remote
else
util_file=$src/git-vault-helpers-remote
fi
if ! [ -f $util_file ]; then
echo "Set VAULT_LIB_DIR $src" >&2
exit 1
}
function log {
echo "`basename $0`: ${@}" 1>&2
}
fi
source $util_file $src
log "Args: $# is $@"
trace "Args: $# is $@"
alias=$1
url=$2
proto=vault
marks_path() {
echo "$GIT_DIR/$proto/marks"
}
obj_id() {
echo "$1" | md5sum | cut -d ' ' -f1
}
obj_list_path() {
echo "$url/.objects.$1"
}
head_path() {
echo "$url/.head.$1"
}
print_caps() {
trace_if 2 "Caps $cmd"
echo "push"
echo "fetch"
echo "refspec refs/heads/*:refs/$proto/$alias/branches/*"
echo "*import-marks $(marks_path)"
echo "*export-marks $(marks_path)"
echo "option"
echo
}
save_option() {
log "Option $@"
trace_if 2 "GIT: Save option $@"
echo "unsupported"
}
list_refs() {
log "List"
if ! [[ -f $(marks_path) ]]; then
[[ -d $GIT_DIR/$proto ]] || mkdir $GIT_DIR/$proto
touch $(marks_path)
fi
if ! find $url -name '.head.*' -exec cat {} \; ; then
echo "? /refs/heads/master"
echo "? /refs/notes/commits"
fi
echo "@refs/heads/master HEAD"
#git for-each-ref --format='? %(refname)' 'refs/heads/'
#echo "@master master"
echo
}
files_to_remove=
on_normal_exit() {
if [ "x$files_to_remove" != "x" ]; then
log "removing"
rm $files_to_remove
fi
}
on_failure() {
on_normal_exit
}
trap on_failure TERM
trap on_normal_exit EXIT
get_temp_file() {
local res=$(mktemp)
files_to_remove="$files_to_remove $res"
echo $res
}
get_saved_object() {
ls $url | sed -e 's/\.\(tree\|blob\|commit\|note\)$//' | sort
}
get_ref_objects() {
git rev-list --objects $1 | cut -d ' ' -f1 | sort
}
add_object() {
local obj_type=$(git cat-file -t $1)
git cat-file $obj_type $1 > $url/$1.$obj_type
}
do_push() {
log "Push $2"
trace_if 2 "GIT: Push $@"
local refs=$2
if [[ ${refs:0:1} == "+" ]]; then
refs=${refs:1}
fi
local delim=$(expr index "$refs" ':')
local from=${refs:0:$(expr delim-1)}
local to=${refs:$delim}
log "$from -> $to"
local obj=$(obj_id $to)
local is_update_head=true
if [ "x$from" == "x" ]; then
log "delete"
warn "TODO: delete $to"
is_update_head=false
else
([[ -d $url ]] || mkdir $url) || error "Can't mkdir $url"
local from_id=$(git rev-parse $from)
trace_if 3 "$from ($from_id) -> $to"
([[ -d $url ]] || mkdir -p $url) || error 15 "Can't mkdir $url"
if [[ $to =~ notes/.* ]]; then
log "Pushing notes"
local notes=$(get_temp_file)
for n in $(git notes list | cut -d ' ' -f1); do
add_object $n
trace_if 3 "Pushing notes"
is_update_head=false
local notes=$(tmp_path notes)
git notes list > $notes
for n in $(cat $notes | cut -d ' ' -f1); do
# TODO remove also notes
push_object $n
done
git notes list > $url/.notes
mv $notes $(notes_list_path)
elif [ "x$(remote_head_id)" == "x$from_id" ]; then
trace_if 3 "Skip $from_id, already taken"
else
local actual_objects=$(get_temp_file)
get_ref_objects $from > $actual_objects
local saved_objects=$(get_temp_file)
get_saved_object > $saved_objects
local obj_type=
for i in $(comm -13 $saved_objects $actual_objects); do
log "add $i "
add_object $i
trace_if 3 "Processing $from_id"
local src_list_prefix=$(save_graph $from)
# objects
local actual_objects=$src_list_prefix.objects
local saved_objects=$(tmp_path objects.saved)
write_saved_objects $saved_objects
process_sorted_ranges $saved_objects $actual_objects \
push_object rm_remote_object
# blobs
local actual_blobs=$src_list_prefix.blobs
local saved_blobs=$(tmp_path blobs.saved)
write_saved_blobs $saved_blobs
process_sorted_ranges $saved_blobs $actual_blobs \
push_vault_blob rm_remote_blob
# tags
local tags_file=$(tmp_path tags)
for tag in $(git tag); do
echo "$tag $(git rev-parse $tag)" >> $tags_file
done
cat $actual_objects > $(obj_list_path $obj)
echo "$(git rev-parse $from) $from" > $(head_path $obj)
mv $tags_file $(tags_list_path)
# record saved objects
rm -f $url/*.objects
rm -f $url/*.blobs
cp $src_list_prefix.blobs $url/
cp $src_list_prefix.objects $url/
fi
fi
if $is_update_head; then
echo "$from_id $from" > $(head_path $obj)
fi
tar cf $url/metadata.tar .git/info/exclude .git/vault.version
echo
}
tag_object() {
git tag $1 $2 || warn "Can't tag $1 $2"
}
do_fetch() {
log "Fetch $@"
# TODO ignore params, just fetch everything
local obj_type=
local obj_id=
for i in $(ls $url); do
obj_id=$(echo "$i" | cut -d '.' -f1)
if ! git cat-file -e $obj_id; then
log "add $i $obj_type"
obj_type=$(echo "$i" | cut -d '.' -f 2)
obj_id=$(git hash-object -t $obj_type -w $url/$i)
fi
trace_if 2 "GIT: Fetch $@"
# TODO now it ignores params, just fetches everything
local saved_objects=$(tmp_path objects.saved)
write_saved_objects_filenames $saved_objects
local obj=
for obj in $(cat $saved_objects); do
fetch_object $obj
done
if [ -f $url/.notes ]; then
for n in $(cat $url/.notes | tr ' ' ':'); do
[ -d .git/blobs ] || (trace_if 2 "Creating blobs dir"; mkdir .git/blobs)
local saved_blobs=$(tmp_path blobs.saved)
write_saved_blobs $saved_blobs
local actual_blobs=$(tmp_path blobs.actual)
write_actual_blobs $actual_blobs
process_sorted_ranges $actual_blobs $saved_blobs \
fetch_vault_blob rm_vault_blob
apply_remote_tags tag_object
local metadata_file=$url/metadata.tar
[ -f $metadata_file ] && (trace_if 2 "Extracting metadata"; tar xf $metadata_file)
local notes_list=$(notes_list_path)
if [ -f $notes_list ]; then
for n in $(cat $notes_list | tr ' ' ':'); do
local delim=$(expr index "$n" ':')
local note=$url/${n:0:$(expr delim-1)}.blob
local note=$(remote_object_path ${n:0:$(expr delim-1)} "blob")
local commit=${n:$delim}
if [ -f $note ]; then
git notes add -f -F $note $commit
else
log "Can't find note $note"
warn "Can't find note $note"
fi
done
else
warn "No notes"
fi
x echo
echo
}
list_for_push() {
log "for push $@"
trace_if 2 "for push $@"
list_refs $@
}
while read cmd; do
trace "GIT CMD: $cmd"
case $cmd in
capabilities)
log "Caps $cmd"
print_caps
#break
;;
......@@ -189,8 +176,7 @@ while read cmd; do
;;
*)
[[ "x$cmd" == "x" ]] && break
log "Cmd: $cmd"
trace "Cmd: $cmd"
;;
esac
done
#!/bin/bash
if [ "x$VAULT_LIB_DIR" == "x" ]; then
echo "Set VAULT_LIB_DIR" >&2
exit 1
fi
src=$(dirname $0)
src=$(cd $src;pwd)
source $VAULT_LIB_DIR/vault-misc $src
obj_id() {
echo "$1" | sha1sum | cut -d ' ' -f1
}
head_path() {
echo "$url/$1.head"
}
notes_list_path() {
echo "$url/notes"
}
tags_list_path() {
echo "$url/tags"
}
print_heads() {
if [ -d $url ]; then
if [ $trace_level -gt 2 ]; then
trace 3 "$(find $url -name '*.head')"
trace 3 "$(find $url -name '*.head' | xargs cat)"
fi
find $url -name '*.head' | xargs cat
else
return 1
fi
}
remote_object_path() {
ensure_param_count_exit_usage $# 2 "Provide ID and type"
echo "$url/$1.$2"
}
remote_object_list() {
ls $url/*.objects
}
has_commit() {
if [ -f $(remote_object_path $1 "commit") ] \
|| (find .git -name '*.objects' -print | xargs grep $1); then
return 0
else
return -1
fi
}
remote_head_id() {
local head_file=$(head_path $obj)
if [ -f $head_file ]; then
cat $head_file | cut -d ' ' -f 1
else
echo
fi
}
list_refs() {
trace_if 2 "GIT: Listing refs"
# TODO verify head is consistent
if ! print_heads; then
trace "Not yet initialized"
echo "? /refs/heads/master"
echo "? /refs/notes/commits"
fi
echo "@refs/heads/master HEAD"
#git for-each-ref --format='? %(refname)' 'refs/heads/'
#echo "@master master"
echo
}
vault_tmp_dir=$(mktemp -d)
normal_cleanup="$normal_cleanup; vault_remote_rm_tmp_dir"
tmp_path() {
echo $vault_tmp_dir/$1
}
vault_remote_rm_tmp_dir() {
if [ "x$vault_tmp_dir" != "x" ]; then
trace_if 2 "removing $vault_tmp_dir"
[[ -d $vault_tmp_dir ]] && rm -rf $vault_tmp_dir
fi
}
if [[ "x$VAULT_LIB_DIR" == "x" ]]; then
VAULT_LIB_DIR=$(dirname $0)
fi
write_saved_objects() {
ensure_param_count_exit_usage $# 1 "Provide destination"
local dst=$1
local obj_re='\.\(tree\|blob\|commit\|note\)$'
ls $url | grep "$obj_re" | sed -e "s/$obj_re//" | sort > $dst
}
write_saved_objects_filenames() {
ensure_param_count_exit_usage $# 1 "Provide destination"
local dst=$1
local obj_re='\.\(tree\|blob\|commit\|note\)$'
ls $url | grep "$obj_re" | sort > $dst
}
apply_remote_object() {
ensure_param_count_exit_usage $# 2 "Provide object and function"
local obj_file=$1
local fn=$2
local delim=$(expr index "$obj_file" '.')
local obj_id=${obj_file:0:$(expr delim-1)}
local obj_type=${obj_file:delim}
fn $obj_id $obj_type
}
apply_remote_tags() {
ensure_param_count_exit_usage $# 1 "Provide function"
local fn=$1
local tags_list=$(tags_list_path)
if [ -f $tags_list ]; then
local delim
local line
for line in $(cat $tags_list | tr ' ' ':'); do
delim=$(expr index "$line" ':')
$fn ${line:0:$(expr delim-1)} ${line:delim}
done
fi
}
write_saved_blobs() {
ensure_param_count_exit_usage $# 1 "Provide destination"
local dst=$1
find $url -name '*.vblob' -exec basename {} .vblob \; | sort > $dst
}
get_local_blob_id() {
ensure_param_count_exit_usage $# 1 "Provide blob path"
local blob_dir=$(basename $(dirname $1))
local blob_file=$(basename $1)
echo "$blob_dir$blob_file"
}
export -f get_local_blob_id
write_actual_blobs() {
ensure_param_count_exit_usage $# 1 "Provide destination"
local dst=$1
find .git/blobs -type f -exec get_local_blob_id {} \; | sort > $dst
}
vault_blob_remote_path() {
echo "$url/$1.vblob"
}
vault_blob_local_path() {
echo ".git/blobs/${1:0:2}/${1:2}"
}
apply_local_blob() {
ensure_param_count_exit_usage $# 2 "Provide id and function"
local obj_id=$(verify_obj_id $1)
local fn=$2
$fn $obj_id .git/blobs/${obj_id:0:2} ${obj_id:2} ${@:3}
}
vault_object_remote_path() {
local obj_id=$1
local obj_type=$2
echo "$url/$obj_id.$obj_type"
}
sort_file() {
local fname=$1
sort -o $fname $fname
}
save_graph() {
trace "Saving graph"
local commit_id=$(git rev-parse $1)
# TODO hard-coded
local skip_blob_re="^Gallery\/"
local out_filename_prefix=$vault_tmp_dir/$commit_id
git rev-list --objects $commit_id \
| gawk -v skip_blob=$skip_blob_re \
-v out_prefix=$out_filename_prefix \
-f $VAULT_LIB_DIR/git-vault-split-refs.awk 2>/dev/null
sort_file $out_filename_prefix.objects
local blobs_list=$out_filename_prefix.blobs
echo -n > $blobs_list
for obj_id in $(cat $out_filename_prefix.blob.objects); do
if [ "x$obj_id" != "x" ] && [ "x$(git cat-file -t $obj_id)" == "xblob" ]; then
local blob_link=$(git cat-file -p $obj_id)
local blob_dir=$(basename $(dirname $blob_link))
local blob_file=$(basename $blob_link)
local blob_id="$blob_dir$blob_file"
if [ $(expr length "$blob_id") -eq 40 ]; then
echo $blob_id >> $blobs_list
else
trace "$(expr length $blob_id) skip not hash $blob_id for $obj_link"
fi
fi
done
sort_file $blobs_list
echo $out_filename_prefix
}
push_vault_blob() {
trace_if 3 "Push blob $1"
local blob_id=$1
local dst=$(vault_blob_remote_path $blob_id)
if ! [ -f $dst ]; then
local src=$(vault_blob_local_path $blob_id)
[ -f $src ] && cp $src $dst || error 11 "No vault blob $src"
fi
}
rm_remote_blob() {
trace_if 3 "Remove blob $1"
local obj_id=$(verify_obj_id $1)
rm -f $url/$obj_id.vblob
}
fetch_blob_create_dir() {
local blob_id=$1
local blob_dir=$2
local dst=$2/$3
local src=$(vault_blob_remote_path $blob_id)
[ -f $src ] || error 30 "No remote blob $src"
if [ -d $blob_dir ]; then
if [ -f $dst ]; then
trace_if 3 "Skip, already exist: $blob_id"
return 0
fi
else
mkdir -p $blob_dir
fi
cp $src $dst || error 12 "Can't copy $src $dst"
}
fetch_vault_blob() {
apply_local_blob $1 fetch_blob_create_dir
}
rm_vault_blob() {
ensure_param_count_exit_usage $# 1 "Provide blob id"
local blob_id=$1
local target=$(vault_blob_local_path $blob_id)
[ -f $target ] && rm -f $target
}
push_object() {
trace_if 3 "Push $1"
local obj_type=$(git cat-file -t $1)
[ "x$obj_type" == "x" ] && error 20 "Can't get obj_type from $1"
local dst=$(vault_object_remote_path $1 $obj_type)
trace_if 3 "push $1 ($obj_type) -> $dst"
[ -f $dst ] || (git cat-file $obj_type $1 > $dst)
}
verify_obj_id() {
[ $(expr length "$1") -eq 40 ] || error 13 "Wrong obj ID: $1"
echo $1
}
rm_remote_object() {
trace_if 3 "Remove $1"
local obj_id=$(verify_obj_id $1)
rm -f $url/$obj_id.*
}
fetch_object() {
local obj_type=$(echo "$1" | cut -d '.' -f 2)
local obj_id=$(echo "$1" | cut -d '.' -f 1)
local dst_id=
trace_if 3 "Fetch $obj_id $obj_type"
if ! git cat-file -e $obj_id 2>/dev/null; then
dst_id=$(git hash-object -t $obj_type -w $url/$1 || error 14 "Can't fetch $1")
fi
}
process_sorted_ranges() {
local before=$1
local after=$2
local on_added=$3
local on_removed=$4
trace_if 2 "process $before vs $after"
for line in $(diff -w $before $after | sed -e 's/\([<>]\) /\1/'); do
trace_if 3 "Process line: $line"
if [ $(expr length "$line") -gt 1 ]; then
local prefix=${line:0:1}
local data=${line:1}
if [ "x$prefix" == "x>" ]; then
$on_added "$data"
elif [ "x$prefix" == "x<" ]; then
$on_removed "$data"
fi
fi
done
}
BEGIN {
if (out_prefix == "") {
print "provide out_prefix path" > "/dev/stderr"
exit 1
}
blob_out = out_prefix ".blob.objects";
hash_out = out_prefix ".objects";
full_out = out_prefix ".revlist";
print > blob_out;
print > hash_out;
print > full_out;
}
$2 ~ /^[A-Za-z0-9_]+\/blobs\// {
if (! skip_blob || $2 !~ skip_blob) {
print $1 >> blob_out
} else {
print "skip " $2 > "/dev/stderr"
}
}
$1 ~ /[0-9a-f]+/ {
git cat-file -t $1
print $1 >> hash_out
}
{
print $0 >> full_out
}
......@@ -8,8 +8,10 @@ root=
export TOP_PID=$$
is_trace=false
trace_level=0
if [ "x$VAULT_TRACE" != "x" ]; then
is_trace=true
is_trace=true
trace_level=$VAULT_TRACE
fi
function trace {
......@@ -18,6 +20,12 @@ function trace {
fi
}
function trace_if {
if $is_trace && [ $trace_level -ge $1 ]; then
echo "`basename $0`: ${@:2}" 1>&2;
fi
}
normal_cleanup="true"
failed_cleanup="true"
......@@ -68,6 +76,7 @@ function error {
on_failure
fi
warn "Killing $TOP_PID"
kill -s INT $TOP_PID
kill -s TERM $TOP_PID
}
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment