diff --git a/rpm/vault.spec b/rpm/vault.spec index 66dda59..e1f764c 100644 --- a/rpm/vault.spec +++ b/rpm/vault.spec @@ -91,6 +91,7 @@ rm -rf $RPM_BUILD_ROOT %{_bindir}/vault %{_bindir}/vault-sync %{_bindir}/vault-resolve +%{_bindir}/git-remote-vault %dir %{tools_dir} %{tools_dir}/* %if 0%{?_with_usersession:1} diff --git a/tools/CMakeLists.txt b/tools/CMakeLists.txt index 1a287b9..2bedcaa 100644 --- a/tools/CMakeLists.txt +++ b/tools/CMakeLists.txt @@ -8,11 +8,20 @@ install( DESTINATION ${TOOLS_DIR} ) +install( + PROGRAMS + git-remote-vault + DESTINATION bin + ) + install( FILES - git-vault-rebase-generate.awk git-vault-rebase-prepare.awk + git-vault-rebase-generate.awk + git-vault-rebase-prepare.awk + git-vault-split-refs.awk vault-misc vault-unit-common + git-vault-helpers-remote DESTINATION ${TOOLS_DIR} ) diff --git a/tools/git-remote-vault b/tools/git-remote-vault index 330b9d3..5305412 100755 --- a/tools/git-remote-vault +++ b/tools/git-remote-vault @@ -1,174 +1,161 @@ #!/bin/bash -function error { - echo "`basename $0`: Error: ${@:1}" 1>&2; +src=$(dirname $0) +src=$(cd $src;pwd) +util_file= +if [ "x$VAULT_LIB_DIR" != "x" ]; then + util_file=$VAULT_LIB_DIR/git-vault-helpers-remote +else + util_file=$src/git-vault-helpers-remote +fi +if ! [ -f $util_file ]; then + echo "Set VAULT_LIB_DIR $src" >&2 exit 1 -} - -function log { - echo "`basename $0`: ${@}" 1>&2 -} +fi +source $util_file $src -log "Args: $# is $@" +trace "Args: $# is $@" alias=$1 url=$2 proto=vault -marks_path() { -echo "$GIT_DIR/$proto/marks" -} - -obj_id() { - echo "$1" | md5sum | cut -d ' ' -f1 -} - -obj_list_path() { - echo "$url/.objects.$1" -} - -head_path() { - echo "$url/.head.$1" -} - print_caps() { + trace_if 2 "Caps $cmd" echo "push" echo "fetch" echo "refspec refs/heads/*:refs/$proto/$alias/branches/*" - echo "*import-marks $(marks_path)" - echo "*export-marks $(marks_path)" echo "option" echo } save_option() { - log "Option $@" + trace_if 2 "GIT: Save option $@" echo "unsupported" } -list_refs() { - log "List" - if ! [[ -f $(marks_path) ]]; then - [[ -d $GIT_DIR/$proto ]] || mkdir $GIT_DIR/$proto - touch $(marks_path) - fi - - if ! find $url -name '.head.*' -exec cat {} \; ; then - echo "? /refs/heads/master" - echo "? /refs/notes/commits" - fi - echo "@refs/heads/master HEAD" - #git for-each-ref --format='? %(refname)' 'refs/heads/' - #echo "@master master" - echo -} - -files_to_remove= - -on_normal_exit() { - if [ "x$files_to_remove" != "x" ]; then - log "removing" - rm $files_to_remove - fi -} -on_failure() { - on_normal_exit -} -trap on_failure TERM -trap on_normal_exit EXIT - -get_temp_file() { - local res=$(mktemp) - files_to_remove="$files_to_remove $res" - echo $res -} - -get_saved_object() { - ls $url | sed -e 's/\.\(tree\|blob\|commit\|note\)$//' | sort -} - -get_ref_objects() { - git rev-list --objects $1 | cut -d ' ' -f1 | sort -} - -add_object() { - local obj_type=$(git cat-file -t $1) - git cat-file $obj_type $1 > $url/$1.$obj_type -} - do_push() { - log "Push $2" + trace_if 2 "GIT: Push $@" local refs=$2 + if [[ ${refs:0:1} == "+" ]]; then + refs=${refs:1} + fi local delim=$(expr index "$refs" ':') local from=${refs:0:$(expr delim-1)} local to=${refs:$delim} - log "$from -> $to" local obj=$(obj_id $to) + local is_update_head=true + if [ "x$from" == "x" ]; then - log "delete" + warn "TODO: delete $to" + is_update_head=false else - ([[ -d $url ]] || mkdir $url) || error "Can't mkdir $url" + local from_id=$(git rev-parse $from) + trace_if 3 "$from ($from_id) -> $to" + ([[ -d $url ]] || mkdir -p $url) || error 15 "Can't mkdir $url" if [[ $to =~ notes/.* ]]; then - log "Pushing notes" - local notes=$(get_temp_file) - for n in $(git notes list | cut -d ' ' -f1); do - add_object $n + trace_if 3 "Pushing notes" + is_update_head=false + local notes=$(tmp_path notes) + git notes list > $notes + for n in $(cat $notes | cut -d ' ' -f1); do + # TODO remove also notes + push_object $n done - git notes list > $url/.notes + mv $notes $(notes_list_path) + elif [ "x$(remote_head_id)" == "x$from_id" ]; then + trace_if 3 "Skip $from_id, already taken" else - local actual_objects=$(get_temp_file) - get_ref_objects $from > $actual_objects - local saved_objects=$(get_temp_file) - get_saved_object > $saved_objects - - local obj_type= - for i in $(comm -13 $saved_objects $actual_objects); do - log "add $i " - add_object $i + trace_if 3 "Processing $from_id" + local src_list_prefix=$(save_graph $from) + + # objects + local actual_objects=$src_list_prefix.objects + local saved_objects=$(tmp_path objects.saved) + write_saved_objects $saved_objects + process_sorted_ranges $saved_objects $actual_objects \ + push_object rm_remote_object + + # blobs + local actual_blobs=$src_list_prefix.blobs + local saved_blobs=$(tmp_path blobs.saved) + write_saved_blobs $saved_blobs + process_sorted_ranges $saved_blobs $actual_blobs \ + push_vault_blob rm_remote_blob + + # tags + local tags_file=$(tmp_path tags) + for tag in $(git tag); do + echo "$tag $(git rev-parse $tag)" >> $tags_file done - cat $actual_objects > $(obj_list_path $obj) - echo "$(git rev-parse $from) $from" > $(head_path $obj) + mv $tags_file $(tags_list_path) + + # record saved objects + rm -f $url/*.objects + rm -f $url/*.blobs + cp $src_list_prefix.blobs $url/ + cp $src_list_prefix.objects $url/ fi fi + if $is_update_head; then + echo "$from_id $from" > $(head_path $obj) + fi tar cf $url/metadata.tar .git/info/exclude .git/vault.version echo } +tag_object() { + git tag $1 $2 || warn "Can't tag $1 $2" +} + do_fetch() { - log "Fetch $@" - # TODO ignore params, just fetch everything - local obj_type= - local obj_id= - for i in $(ls $url); do - obj_id=$(echo "$i" | cut -d '.' -f1) - if ! git cat-file -e $obj_id; then - log "add $i $obj_type" - obj_type=$(echo "$i" | cut -d '.' -f 2) - obj_id=$(git hash-object -t $obj_type -w $url/$i) - fi + trace_if 2 "GIT: Fetch $@" + # TODO now it ignores params, just fetches everything + local saved_objects=$(tmp_path objects.saved) + write_saved_objects_filenames $saved_objects + local obj= + for obj in $(cat $saved_objects); do + fetch_object $obj done - if [ -f $url/.notes ]; then - for n in $(cat $url/.notes | tr ' ' ':'); do + + [ -d .git/blobs ] || (trace_if 2 "Creating blobs dir"; mkdir .git/blobs) + local saved_blobs=$(tmp_path blobs.saved) + write_saved_blobs $saved_blobs + local actual_blobs=$(tmp_path blobs.actual) + write_actual_blobs $actual_blobs + + process_sorted_ranges $actual_blobs $saved_blobs \ + fetch_vault_blob rm_vault_blob + + apply_remote_tags tag_object + local metadata_file=$url/metadata.tar + [ -f $metadata_file ] && (trace_if 2 "Extracting metadata"; tar xf $metadata_file) + local notes_list=$(notes_list_path) + if [ -f $notes_list ]; then + for n in $(cat $notes_list | tr ' ' ':'); do local delim=$(expr index "$n" ':') - local note=$url/${n:0:$(expr delim-1)}.blob + local note=$(remote_object_path ${n:0:$(expr delim-1)} "blob") local commit=${n:$delim} if [ -f $note ]; then git notes add -f -F $note $commit else - log "Can't find note $note" + warn "Can't find note $note" fi done + else + warn "No notes" fi -x echo + echo } list_for_push() { - log "for push $@" + trace_if 2 "for push $@" list_refs $@ } + while read cmd; do + trace "GIT CMD: $cmd" case $cmd in capabilities) - log "Caps $cmd" print_caps #break ;; @@ -189,8 +176,7 @@ while read cmd; do ;; *) [[ "x$cmd" == "x" ]] && break - log "Cmd: $cmd" + trace "Cmd: $cmd" ;; esac done - diff --git a/tools/git-vault-helpers-remote b/tools/git-vault-helpers-remote new file mode 100644 index 0000000..4489e39 --- /dev/null +++ b/tools/git-vault-helpers-remote @@ -0,0 +1,307 @@ +#!/bin/bash + +if [ "x$VAULT_LIB_DIR" == "x" ]; then + echo "Set VAULT_LIB_DIR" >&2 + exit 1 +fi + +src=$(dirname $0) +src=$(cd $src;pwd) +source $VAULT_LIB_DIR/vault-misc $src + +obj_id() { + echo "$1" | sha1sum | cut -d ' ' -f1 +} + +head_path() { + echo "$url/$1.head" +} + +notes_list_path() { + echo "$url/notes" +} + +tags_list_path() { + echo "$url/tags" +} + +print_heads() { + if [ -d $url ]; then + if [ $trace_level -gt 2 ]; then + trace 3 "$(find $url -name '*.head')" + trace 3 "$(find $url -name '*.head' | xargs cat)" + fi + find $url -name '*.head' | xargs cat + else + return 1 + fi +} + +remote_object_path() { + ensure_param_count_exit_usage $# 2 "Provide ID and type" + echo "$url/$1.$2" +} + +remote_object_list() { + ls $url/*.objects +} + +has_commit() { + if [ -f $(remote_object_path $1 "commit") ] \ + || (find .git -name '*.objects' -print | xargs grep $1); then + return 0 + else + return -1 + fi +} + +remote_head_id() { + local head_file=$(head_path $obj) + if [ -f $head_file ]; then + cat $head_file | cut -d ' ' -f 1 + else + echo + fi +} + +list_refs() { + trace_if 2 "GIT: Listing refs" + # TODO verify head is consistent + if ! print_heads; then + trace "Not yet initialized" + echo "? /refs/heads/master" + echo "? /refs/notes/commits" + fi + echo "@refs/heads/master HEAD" + #git for-each-ref --format='? %(refname)' 'refs/heads/' + #echo "@master master" + echo +} + +vault_tmp_dir=$(mktemp -d) + +normal_cleanup="$normal_cleanup; vault_remote_rm_tmp_dir" +tmp_path() { + echo $vault_tmp_dir/$1 +} + +vault_remote_rm_tmp_dir() { + if [ "x$vault_tmp_dir" != "x" ]; then + trace_if 2 "removing $vault_tmp_dir" + [[ -d $vault_tmp_dir ]] && rm -rf $vault_tmp_dir + fi +} + +if [[ "x$VAULT_LIB_DIR" == "x" ]]; then + VAULT_LIB_DIR=$(dirname $0) +fi + +write_saved_objects() { + ensure_param_count_exit_usage $# 1 "Provide destination" + local dst=$1 + local obj_re='\.\(tree\|blob\|commit\|note\)$' + ls $url | grep "$obj_re" | sed -e "s/$obj_re//" | sort > $dst +} + +write_saved_objects_filenames() { + ensure_param_count_exit_usage $# 1 "Provide destination" + local dst=$1 + local obj_re='\.\(tree\|blob\|commit\|note\)$' + ls $url | grep "$obj_re" | sort > $dst +} + +apply_remote_object() { + ensure_param_count_exit_usage $# 2 "Provide object and function" + local obj_file=$1 + local fn=$2 + local delim=$(expr index "$obj_file" '.') + local obj_id=${obj_file:0:$(expr delim-1)} + local obj_type=${obj_file:delim} + fn $obj_id $obj_type +} + +apply_remote_tags() { + ensure_param_count_exit_usage $# 1 "Provide function" + local fn=$1 + local tags_list=$(tags_list_path) + if [ -f $tags_list ]; then + local delim + local line + for line in $(cat $tags_list | tr ' ' ':'); do + delim=$(expr index "$line" ':') + $fn ${line:0:$(expr delim-1)} ${line:delim} + done + fi +} + +write_saved_blobs() { + ensure_param_count_exit_usage $# 1 "Provide destination" + local dst=$1 + find $url -name '*.vblob' -exec basename {} .vblob \; | sort > $dst +} + +get_local_blob_id() { + ensure_param_count_exit_usage $# 1 "Provide blob path" + local blob_dir=$(basename $(dirname $1)) + local blob_file=$(basename $1) + echo "$blob_dir$blob_file" +} + +export -f get_local_blob_id + +write_actual_blobs() { + ensure_param_count_exit_usage $# 1 "Provide destination" + local dst=$1 + find .git/blobs -type f -exec get_local_blob_id {} \; | sort > $dst +} + +vault_blob_remote_path() { + echo "$url/$1.vblob" +} + +vault_blob_local_path() { + echo ".git/blobs/${1:0:2}/${1:2}" +} + +apply_local_blob() { + ensure_param_count_exit_usage $# 2 "Provide id and function" + local obj_id=$(verify_obj_id $1) + local fn=$2 + $fn $obj_id .git/blobs/${obj_id:0:2} ${obj_id:2} ${@:3} +} + +vault_object_remote_path() { + local obj_id=$1 + local obj_type=$2 + echo "$url/$obj_id.$obj_type" +} + +sort_file() { + local fname=$1 + sort -o $fname $fname +} + +save_graph() { + trace "Saving graph" + local commit_id=$(git rev-parse $1) + # TODO hard-coded + local skip_blob_re="^Gallery\/" + local out_filename_prefix=$vault_tmp_dir/$commit_id + git rev-list --objects $commit_id \ + | gawk -v skip_blob=$skip_blob_re \ + -v out_prefix=$out_filename_prefix \ + -f $VAULT_LIB_DIR/git-vault-split-refs.awk 2>/dev/null + sort_file $out_filename_prefix.objects + local blobs_list=$out_filename_prefix.blobs + echo -n > $blobs_list + for obj_id in $(cat $out_filename_prefix.blob.objects); do + if [ "x$obj_id" != "x" ] && [ "x$(git cat-file -t $obj_id)" == "xblob" ]; then + local blob_link=$(git cat-file -p $obj_id) + local blob_dir=$(basename $(dirname $blob_link)) + local blob_file=$(basename $blob_link) + local blob_id="$blob_dir$blob_file" + if [ $(expr length "$blob_id") -eq 40 ]; then + echo $blob_id >> $blobs_list + else + trace "$(expr length $blob_id) skip not hash $blob_id for $obj_link" + fi + fi + done + sort_file $blobs_list + echo $out_filename_prefix +} + +push_vault_blob() { + trace_if 3 "Push blob $1" + local blob_id=$1 + local dst=$(vault_blob_remote_path $blob_id) + if ! [ -f $dst ]; then + local src=$(vault_blob_local_path $blob_id) + [ -f $src ] && cp $src $dst || error 11 "No vault blob $src" + fi +} + +rm_remote_blob() { + trace_if 3 "Remove blob $1" + local obj_id=$(verify_obj_id $1) + rm -f $url/$obj_id.vblob +} + +fetch_blob_create_dir() { + local blob_id=$1 + local blob_dir=$2 + local dst=$2/$3 + local src=$(vault_blob_remote_path $blob_id) + [ -f $src ] || error 30 "No remote blob $src" + if [ -d $blob_dir ]; then + if [ -f $dst ]; then + trace_if 3 "Skip, already exist: $blob_id" + return 0 + fi + else + mkdir -p $blob_dir + fi + cp $src $dst || error 12 "Can't copy $src $dst" +} + +fetch_vault_blob() { + apply_local_blob $1 fetch_blob_create_dir +} + +rm_vault_blob() { + ensure_param_count_exit_usage $# 1 "Provide blob id" + local blob_id=$1 + local target=$(vault_blob_local_path $blob_id) + [ -f $target ] && rm -f $target +} + +push_object() { + trace_if 3 "Push $1" + local obj_type=$(git cat-file -t $1) + [ "x$obj_type" == "x" ] && error 20 "Can't get obj_type from $1" + local dst=$(vault_object_remote_path $1 $obj_type) + trace_if 3 "push $1 ($obj_type) -> $dst" + [ -f $dst ] || (git cat-file $obj_type $1 > $dst) +} + +verify_obj_id() { + [ $(expr length "$1") -eq 40 ] || error 13 "Wrong obj ID: $1" + echo $1 +} + +rm_remote_object() { + trace_if 3 "Remove $1" + local obj_id=$(verify_obj_id $1) + rm -f $url/$obj_id.* +} + +fetch_object() { + local obj_type=$(echo "$1" | cut -d '.' -f 2) + local obj_id=$(echo "$1" | cut -d '.' -f 1) + local dst_id= + trace_if 3 "Fetch $obj_id $obj_type" + if ! git cat-file -e $obj_id 2>/dev/null; then + dst_id=$(git hash-object -t $obj_type -w $url/$1 || error 14 "Can't fetch $1") + fi +} + +process_sorted_ranges() { + local before=$1 + local after=$2 + local on_added=$3 + local on_removed=$4 + + trace_if 2 "process $before vs $after" + for line in $(diff -w $before $after | sed -e 's/\([<>]\) /\1/'); do + trace_if 3 "Process line: $line" + if [ $(expr length "$line") -gt 1 ]; then + local prefix=${line:0:1} + local data=${line:1} + if [ "x$prefix" == "x>" ]; then + $on_added "$data" + elif [ "x$prefix" == "x<" ]; then + $on_removed "$data" + fi + fi + done +} diff --git a/tools/git-vault-split-refs.awk b/tools/git-vault-split-refs.awk new file mode 100644 index 0000000..f2a743d --- /dev/null +++ b/tools/git-vault-split-refs.awk @@ -0,0 +1,29 @@ +BEGIN { + if (out_prefix == "") { + print "provide out_prefix path" > "/dev/stderr" + exit 1 + } + blob_out = out_prefix ".blob.objects"; + hash_out = out_prefix ".objects"; + full_out = out_prefix ".revlist"; + print > blob_out; + print > hash_out; + print > full_out; +} + +$2 ~ /^[A-Za-z0-9_]+\/blobs\// { + if (! skip_blob || $2 !~ skip_blob) { + print $1 >> blob_out + } else { + print "skip " $2 > "/dev/stderr" + } +} + +$1 ~ /[0-9a-f]+/ { + git cat-file -t $1 + print $1 >> hash_out +} + +{ + print $0 >> full_out +} diff --git a/tools/vault-misc b/tools/vault-misc index 7245461..bbad800 100644 --- a/tools/vault-misc +++ b/tools/vault-misc @@ -8,8 +8,10 @@ root= export TOP_PID=$$ is_trace=false +trace_level=0 if [ "x$VAULT_TRACE" != "x" ]; then - is_trace=true + is_trace=true + trace_level=$VAULT_TRACE fi function trace { @@ -18,6 +20,12 @@ function trace { fi } +function trace_if { + if $is_trace && [ $trace_level -ge $1 ]; then + echo "`basename $0`: ${@:2}" 1>&2; + fi +} + normal_cleanup="true" failed_cleanup="true" @@ -68,6 +76,7 @@ function error { on_failure fi warn "Killing $TOP_PID" + kill -s INT $TOP_PID kill -s TERM $TOP_PID }