Updated build script to make reproducible builds

* fixed mk-rel.sh; put all release artifiacts in releases/ dir
* updated vt-proto dependencies

.gitignore

@@ -31,4 +31,4 @@ sigtool
 *.pub
 *.key
 *.sig
-
+releases/*

build

@@ -19,6 +19,7 @@ Progs="src:sigtool"
 # e.g. src/foo/a.proto
 Protobufs="internal/pb/hdr.proto"
 
+#set -x
 
 # -- DO NOT CHANGE ANYTHING AFTER THIS --
 
@@ -31,6 +32,7 @@ Prodver=""
 Repover=""
 Verbose=0
 Go=`which go`
+Bindir=$PWD/bin
 
 die() {
     echo "$Z: $@" 1>&2
@@ -51,8 +53,14 @@ esac
 getvcs_version() {
     local rev=
     local prodv=
+    local git=`which git`
+    local hg=`which hg`
 
-    if [ -d "./.hg" ]; then
+    if [ -n "$git" ]; then
+        local xrev=$(git describe --always --dirty --long --abbrev=12) || exit 1
+        rev="git:$xrev"
+        prodv=$(git tag --list | sort -V | tail -1)
+    elif [ -n "$hg" ]; then
         local xrev=$(hg id --id) || exit 1
         local brev=${xrev%+}
         if [ "$brev" != "$xrev" ]; then
@@ -61,13 +69,9 @@ getvcs_version() {
             rev="hg:${brev}"
         fi
         prodv=$(hg log -r "branch(stable) and tag()" -T "{tags}\n" | sort -V | tail -1)
-    elif [ -d "./.git" ]; then
-        local xrev=$(git describe --always --dirty --long --abbrev=12) || exit 1
-        rev="git:$xrev"
-        prodv=$(git tag --list | sort -V | tail -1)
     else
+        warn "no git or hg found; can't get VCS info"
         rev="UNKNOWN-VER"
-        warn "Can't find version info"
     fi
 
     [ -n "$Prodver" ] && prodv=$Prodver
@@ -96,8 +100,6 @@ $0 - A Go production build tool that adds git-repository information,
 product version, build-timestamp etc. It supports cross-compilation,
 static linking and generating protobuf output.
 
-If needed, it uses the gogo-slick protobuf compiler [github.com/gogo/protobuf].
-
 Build output is in bin/\$OS-\$CPU for a given OS, CPU combination.
 
 Usage: $0
@@ -109,10 +111,11 @@ And, PROGS is one or more go programs.
 With no arguments, $0 builds: $pstr
 
 The repository's latest tag is used as the default version of the software being
-built.
+built. The current repository version is $Repover.
 
 Options:
     -h, --help          Show this help message and quit
+    -b D, --bindir=D    Put the binaries in the directory 'D' [$Bindir]
     -s, --static        Build a statically linked binary [False]
     -V N, --version=N   Use 'N' as the product version string [$Prodver]
     -a X, --arch=X      Cross compile for OS-CPU 'X' [$hostos-$hostcpu]
@@ -195,6 +198,14 @@ do
             ac_prev=Arch
             ;;
 
+        -b|--bindir)
+            ac_prev=Bindir
+            ;;
+
+        --bindir=*)
+            Bindir=$ac_optarg
+            ;;
+
         --version=*)
             Prodver=$ac_optarg
             ;;
@@ -376,11 +387,11 @@ fi
 
 
 # This is where build outputs go
-Bindir=$PWD/bin/$cross
+Outdir=$Bindir/$cross
-Hostbindir=$PWD/bin/$hostos-$hostcpu
+Hostbindir=$Bindir/$hostos-$hostcpu
 export PATH=$Hostbindir:$PATH
 
-[ -d $Bindir ]     || mkdir -p $Bindir
+[ -d $Outdir ]     || mkdir -p $Outdir
 [ -d $Hostbindir ] || mkdir -p $Hostbindir
 
 
@@ -395,7 +406,7 @@ fi
 repover="main.RepoVersion=$Repover"
 prodver="main.ProductVersion=$Prodver"
 date="main.Buildtime=`date -u '+%Y-%m-%dT%H:%M.%SZ'`"
-ldflags="-ldflags \"-X $repover -X $prodver -X $date $ldflags\""
+ldflags="-ldflags \"-X $repover -X $prodver -X $date $ldflags -buildid=\""
 vflag=""
 
 [ $Verbose -gt 0 ] && vflag="-v"
@@ -424,6 +435,8 @@ case $Tool in
             all="$@"
         fi
 
+        [ -z "$all" ] && die "No programs specified. Try '$Z --help'"
+
         echo "Building $Prodver ($Repover), $cross $msg .."
 
         for p in $all; do
@@ -442,7 +455,7 @@ case $Tool in
             fi
 
             echo "   $dir: $out .. "
-            $e eval $Go build $vflag -o $Bindir/$out $isuffix "$ldflags" ./$dir || exit 1
+            $e eval $Go build $vflag -trimpath -o $Outdir/$out $isuffix "$ldflags" ./$dir || exit 1
         done
         ;;
 esac

go.mod

@@ -7,6 +7,7 @@ require (
 	github.com/opencoff/go-mmap v0.1.2
 	github.com/opencoff/go-utils v0.9.3
 	github.com/opencoff/pflag v1.0.6-sh1
+	github.com/planetscale/vtprotobuf v0.6.0
 	golang.org/x/crypto v0.21.0
 	google.golang.org/protobuf v1.33.0
 	gopkg.in/yaml.v2 v2.4.0

go.sum

@@ -8,6 +8,8 @@ github.com/opencoff/go-utils v0.9.3 h1:Gdx1uB6QZ9/8FNU4h+94xGJ+onu8bQLRE4gyKRrrD
 github.com/opencoff/go-utils v0.9.3/go.mod h1:nNx572v21m3AkY9JomSXUG8iruY56Fm2zceNKNJDJaU=
 github.com/opencoff/pflag v1.0.6-sh1 h1:6RO8GgnpH928yu6earGDD01FnFT//bDJ1hCovcVVqY4=
 github.com/opencoff/pflag v1.0.6-sh1/go.mod h1:2bXtpAD/5h/2LarkbsRwiUxqnvB1nZBzn9Xjad1P41A=
+github.com/planetscale/vtprotobuf v0.6.0 h1:nBeETjudeJ5ZgBHUz1fVHvbqUKnYOXNhsIEabROxmNA=
+github.com/planetscale/vtprotobuf v0.6.0/go.mod h1:t/avpk3KcrXxUnYOhZhMXJlSEyie6gQbtLq5NM3loB8=
 golang.org/x/crypto v0.21.0 h1:X31++rzVUdKhX5sWmSOFZxx8UW/ldWx55cbf08iNAMA=
 golang.org/x/crypto v0.21.0/go.mod h1:0BP7YvVV9gBbVKyeTG0Gyn+gZm94bibOW5BjDEYAOMs=
 golang.org/x/sys v0.18.0 h1:DBdB3niSjOA/O0blCZBqDefyWNYveAYMNF1Wum0DYQ4=

internal/pb/hdr.pb.go

@@ -1,6 +1,6 @@
 // Code generated by protoc-gen-go. DO NOT EDIT.
 // versions:
-// 	protoc-gen-go v1.28.1
+// 	protoc-gen-go v1.32.0
 // 	protoc        v3.21.12
 // source: internal/pb/hdr.proto
 

internal/pb/hdr_vtproto.pb.go

@@ -1,14 +1,14 @@
 // Code generated by protoc-gen-go-vtproto. DO NOT EDIT.
-// protoc-gen-go-vtproto version: v0.5.0
+// protoc-gen-go-vtproto version: v0.6.0
 // source: internal/pb/hdr.proto
 
 package pb
 
 import (
 	fmt "fmt"
+	protohelpers "github.com/planetscale/vtprotobuf/protohelpers"
 	protoimpl "google.golang.org/protobuf/runtime/protoimpl"
 	io "io"
-	bits "math/bits"
 )
 
 const (
@@ -55,7 +55,7 @@ func (m *Header) MarshalToSizedBufferVT(dAtA []byte) (int, error) {
 				return 0, err
 			}
 			i -= size
-			i = encodeVarint(dAtA, i, uint64(size))
+			i = protohelpers.EncodeVarint(dAtA, i, uint64(size))
 			i--
 			dAtA[i] = 0x2a
 		}
@@ -63,26 +63,26 @@ func (m *Header) MarshalToSizedBufferVT(dAtA []byte) (int, error) {
 	if len(m.Sender) > 0 {
 		i -= len(m.Sender)
 		copy(dAtA[i:], m.Sender)
-		i = encodeVarint(dAtA, i, uint64(len(m.Sender)))
+		i = protohelpers.EncodeVarint(dAtA, i, uint64(len(m.Sender)))
 		i--
 		dAtA[i] = 0x22
 	}
 	if len(m.Pk) > 0 {
 		i -= len(m.Pk)
 		copy(dAtA[i:], m.Pk)
-		i = encodeVarint(dAtA, i, uint64(len(m.Pk)))
+		i = protohelpers.EncodeVarint(dAtA, i, uint64(len(m.Pk)))
 		i--
 		dAtA[i] = 0x1a
 	}
 	if len(m.Salt) > 0 {
 		i -= len(m.Salt)
 		copy(dAtA[i:], m.Salt)
-		i = encodeVarint(dAtA, i, uint64(len(m.Salt)))
+		i = protohelpers.EncodeVarint(dAtA, i, uint64(len(m.Salt)))
 		i--
 		dAtA[i] = 0x12
 	}
 	if m.ChunkSize != 0 {
-		i = encodeVarint(dAtA, i, uint64(m.ChunkSize))
+		i = protohelpers.EncodeVarint(dAtA, i, uint64(m.ChunkSize))
 		i--
 		dAtA[i] = 0x8
 	}
@@ -122,31 +122,20 @@ func (m *WrappedKey) MarshalToSizedBufferVT(dAtA []byte) (int, error) {
 	if len(m.Nonce) > 0 {
 		i -= len(m.Nonce)
 		copy(dAtA[i:], m.Nonce)
-		i = encodeVarint(dAtA, i, uint64(len(m.Nonce)))
+		i = protohelpers.EncodeVarint(dAtA, i, uint64(len(m.Nonce)))
 		i--
 		dAtA[i] = 0x12
 	}
 	if len(m.DKey) > 0 {
 		i -= len(m.DKey)
 		copy(dAtA[i:], m.DKey)
-		i = encodeVarint(dAtA, i, uint64(len(m.DKey)))
+		i = protohelpers.EncodeVarint(dAtA, i, uint64(len(m.DKey)))
 		i--
 		dAtA[i] = 0xa
 	}
 	return len(dAtA) - i, nil
 }
 
-func encodeVarint(dAtA []byte, offset int, v uint64) int {
-	offset -= sov(v)
-	base := offset
-	for v >= 1<<7 {
-		dAtA[offset] = uint8(v&0x7f | 0x80)
-		v >>= 7
-		offset++
-	}
-	dAtA[offset] = uint8(v)
-	return base
-}
 func (m *Header) SizeVT() (n int) {
 	if m == nil {
 		return 0
@@ -154,24 +143,24 @@ func (m *Header) SizeVT() (n int) {
 	var l int
 	_ = l
 	if m.ChunkSize != 0 {
-		n += 1 + sov(uint64(m.ChunkSize))
+		n += 1 + protohelpers.SizeOfVarint(uint64(m.ChunkSize))
 	}
 	l = len(m.Salt)
 	if l > 0 {
-		n += 1 + l + sov(uint64(l))
+		n += 1 + l + protohelpers.SizeOfVarint(uint64(l))
 	}
 	l = len(m.Pk)
 	if l > 0 {
-		n += 1 + l + sov(uint64(l))
+		n += 1 + l + protohelpers.SizeOfVarint(uint64(l))
 	}
 	l = len(m.Sender)
 	if l > 0 {
-		n += 1 + l + sov(uint64(l))
+		n += 1 + l + protohelpers.SizeOfVarint(uint64(l))
 	}
 	if len(m.Keys) > 0 {
 		for _, e := range m.Keys {
 			l = e.SizeVT()
-			n += 1 + l + sov(uint64(l))
+			n += 1 + l + protohelpers.SizeOfVarint(uint64(l))
 		}
 	}
 	n += len(m.unknownFields)
@@ -186,22 +175,16 @@ func (m *WrappedKey) SizeVT() (n int) {
 	_ = l
 	l = len(m.DKey)
 	if l > 0 {
-		n += 1 + l + sov(uint64(l))
+		n += 1 + l + protohelpers.SizeOfVarint(uint64(l))
 	}
 	l = len(m.Nonce)
 	if l > 0 {
-		n += 1 + l + sov(uint64(l))
+		n += 1 + l + protohelpers.SizeOfVarint(uint64(l))
 	}
 	n += len(m.unknownFields)
 	return n
 }
 
-func sov(x uint64) (n int) {
-	return (bits.Len64(x|1) + 6) / 7
-}
-func soz(x uint64) (n int) {
-	return sov(uint64((x << 1) ^ uint64((int64(x) >> 63))))
-}
 func (m *Header) UnmarshalVT(dAtA []byte) error {
 	l := len(dAtA)
 	iNdEx := 0
@@ -210,7 +193,7 @@ func (m *Header) UnmarshalVT(dAtA []byte) error {
 		var wire uint64
 		for shift := uint(0); ; shift += 7 {
 			if shift >= 64 {
-				return ErrIntOverflow
+				return protohelpers.ErrIntOverflow
 			}
 			if iNdEx >= l {
 				return io.ErrUnexpectedEOF
@@ -238,7 +221,7 @@ func (m *Header) UnmarshalVT(dAtA []byte) error {
 			m.ChunkSize = 0
 			for shift := uint(0); ; shift += 7 {
 				if shift >= 64 {
-					return ErrIntOverflow
+					return protohelpers.ErrIntOverflow
 				}
 				if iNdEx >= l {
 					return io.ErrUnexpectedEOF
@@ -257,7 +240,7 @@ func (m *Header) UnmarshalVT(dAtA []byte) error {
 			var byteLen int
 			for shift := uint(0); ; shift += 7 {
 				if shift >= 64 {
-					return ErrIntOverflow
+					return protohelpers.ErrIntOverflow
 				}
 				if iNdEx >= l {
 					return io.ErrUnexpectedEOF
@@ -270,11 +253,11 @@ func (m *Header) UnmarshalVT(dAtA []byte) error {
 				}
 			}
 			if byteLen < 0 {
-				return ErrInvalidLength
+				return protohelpers.ErrInvalidLength
 			}
 			postIndex := iNdEx + byteLen
 			if postIndex < 0 {
-				return ErrInvalidLength
+				return protohelpers.ErrInvalidLength
 			}
 			if postIndex > l {
 				return io.ErrUnexpectedEOF
@@ -291,7 +274,7 @@ func (m *Header) UnmarshalVT(dAtA []byte) error {
 			var byteLen int
 			for shift := uint(0); ; shift += 7 {
 				if shift >= 64 {
-					return ErrIntOverflow
+					return protohelpers.ErrIntOverflow
 				}
 				if iNdEx >= l {
 					return io.ErrUnexpectedEOF
@@ -304,11 +287,11 @@ func (m *Header) UnmarshalVT(dAtA []byte) error {
 				}
 			}
 			if byteLen < 0 {
-				return ErrInvalidLength
+				return protohelpers.ErrInvalidLength
 			}
 			postIndex := iNdEx + byteLen
 			if postIndex < 0 {
-				return ErrInvalidLength
+				return protohelpers.ErrInvalidLength
 			}
 			if postIndex > l {
 				return io.ErrUnexpectedEOF
@@ -325,7 +308,7 @@ func (m *Header) UnmarshalVT(dAtA []byte) error {
 			var byteLen int
 			for shift := uint(0); ; shift += 7 {
 				if shift >= 64 {
-					return ErrIntOverflow
+					return protohelpers.ErrIntOverflow
 				}
 				if iNdEx >= l {
 					return io.ErrUnexpectedEOF
@@ -338,11 +321,11 @@ func (m *Header) UnmarshalVT(dAtA []byte) error {
 				}
 			}
 			if byteLen < 0 {
-				return ErrInvalidLength
+				return protohelpers.ErrInvalidLength
 			}
 			postIndex := iNdEx + byteLen
 			if postIndex < 0 {
-				return ErrInvalidLength
+				return protohelpers.ErrInvalidLength
 			}
 			if postIndex > l {
 				return io.ErrUnexpectedEOF
@@ -359,7 +342,7 @@ func (m *Header) UnmarshalVT(dAtA []byte) error {
 			var msglen int
 			for shift := uint(0); ; shift += 7 {
 				if shift >= 64 {
-					return ErrIntOverflow
+					return protohelpers.ErrIntOverflow
 				}
 				if iNdEx >= l {
 					return io.ErrUnexpectedEOF
@@ -372,11 +355,11 @@ func (m *Header) UnmarshalVT(dAtA []byte) error {
 				}
 			}
 			if msglen < 0 {
-				return ErrInvalidLength
+				return protohelpers.ErrInvalidLength
 			}
 			postIndex := iNdEx + msglen
 			if postIndex < 0 {
-				return ErrInvalidLength
+				return protohelpers.ErrInvalidLength
 			}
 			if postIndex > l {
 				return io.ErrUnexpectedEOF
@@ -388,12 +371,12 @@ func (m *Header) UnmarshalVT(dAtA []byte) error {
 			iNdEx = postIndex
 		default:
 			iNdEx = preIndex
-			skippy, err := skip(dAtA[iNdEx:])
+			skippy, err := protohelpers.Skip(dAtA[iNdEx:])
 			if err != nil {
 				return err
 			}
 			if (skippy < 0) || (iNdEx+skippy) < 0 {
-				return ErrInvalidLength
+				return protohelpers.ErrInvalidLength
 			}
 			if (iNdEx + skippy) > l {
 				return io.ErrUnexpectedEOF
@@ -416,7 +399,7 @@ func (m *WrappedKey) UnmarshalVT(dAtA []byte) error {
 		var wire uint64
 		for shift := uint(0); ; shift += 7 {
 			if shift >= 64 {
-				return ErrIntOverflow
+				return protohelpers.ErrIntOverflow
 			}
 			if iNdEx >= l {
 				return io.ErrUnexpectedEOF
@@ -444,7 +427,7 @@ func (m *WrappedKey) UnmarshalVT(dAtA []byte) error {
 			var byteLen int
 			for shift := uint(0); ; shift += 7 {
 				if shift >= 64 {
-					return ErrIntOverflow
+					return protohelpers.ErrIntOverflow
 				}
 				if iNdEx >= l {
 					return io.ErrUnexpectedEOF
@@ -457,11 +440,11 @@ func (m *WrappedKey) UnmarshalVT(dAtA []byte) error {
 				}
 			}
 			if byteLen < 0 {
-				return ErrInvalidLength
+				return protohelpers.ErrInvalidLength
 			}
 			postIndex := iNdEx + byteLen
 			if postIndex < 0 {
-				return ErrInvalidLength
+				return protohelpers.ErrInvalidLength
 			}
 			if postIndex > l {
 				return io.ErrUnexpectedEOF
@@ -478,7 +461,7 @@ func (m *WrappedKey) UnmarshalVT(dAtA []byte) error {
 			var byteLen int
 			for shift := uint(0); ; shift += 7 {
 				if shift >= 64 {
-					return ErrIntOverflow
+					return protohelpers.ErrIntOverflow
 				}
 				if iNdEx >= l {
 					return io.ErrUnexpectedEOF
@@ -491,11 +474,11 @@ func (m *WrappedKey) UnmarshalVT(dAtA []byte) error {
 				}
 			}
 			if byteLen < 0 {
-				return ErrInvalidLength
+				return protohelpers.ErrInvalidLength
 			}
 			postIndex := iNdEx + byteLen
 			if postIndex < 0 {
-				return ErrInvalidLength
+				return protohelpers.ErrInvalidLength
 			}
 			if postIndex > l {
 				return io.ErrUnexpectedEOF
@@ -507,12 +490,12 @@ func (m *WrappedKey) UnmarshalVT(dAtA []byte) error {
 			iNdEx = postIndex
 		default:
 			iNdEx = preIndex
-			skippy, err := skip(dAtA[iNdEx:])
+			skippy, err := protohelpers.Skip(dAtA[iNdEx:])
 			if err != nil {
 				return err
 			}
 			if (skippy < 0) || (iNdEx+skippy) < 0 {
-				return ErrInvalidLength
+				return protohelpers.ErrInvalidLength
 			}
 			if (iNdEx + skippy) > l {
 				return io.ErrUnexpectedEOF
@@ -527,88 +510,3 @@ func (m *WrappedKey) UnmarshalVT(dAtA []byte) error {
 	}
 	return nil
 }
-
-func skip(dAtA []byte) (n int, err error) {
-	l := len(dAtA)
-	iNdEx := 0
-	depth := 0
-	for iNdEx < l {
-		var wire uint64
-		for shift := uint(0); ; shift += 7 {
-			if shift >= 64 {
-				return 0, ErrIntOverflow
-			}
-			if iNdEx >= l {
-				return 0, io.ErrUnexpectedEOF
-			}
-			b := dAtA[iNdEx]
-			iNdEx++
-			wire |= (uint64(b) & 0x7F) << shift
-			if b < 0x80 {
-				break
-			}
-		}
-		wireType := int(wire & 0x7)
-		switch wireType {
-		case 0:
-			for shift := uint(0); ; shift += 7 {
-				if shift >= 64 {
-					return 0, ErrIntOverflow
-				}
-				if iNdEx >= l {
-					return 0, io.ErrUnexpectedEOF
-				}
-				iNdEx++
-				if dAtA[iNdEx-1] < 0x80 {
-					break
-				}
-			}
-		case 1:
-			iNdEx += 8
-		case 2:
-			var length int
-			for shift := uint(0); ; shift += 7 {
-				if shift >= 64 {
-					return 0, ErrIntOverflow
-				}
-				if iNdEx >= l {
-					return 0, io.ErrUnexpectedEOF
-				}
-				b := dAtA[iNdEx]
-				iNdEx++
-				length |= (int(b) & 0x7F) << shift
-				if b < 0x80 {
-					break
-				}
-			}
-			if length < 0 {
-				return 0, ErrInvalidLength
-			}
-			iNdEx += length
-		case 3:
-			depth++
-		case 4:
-			if depth == 0 {
-				return 0, ErrUnexpectedEndOfGroup
-			}
-			depth--
-		case 5:
-			iNdEx += 4
-		default:
-			return 0, fmt.Errorf("proto: illegal wireType %d", wireType)
-		}
-		if iNdEx < 0 {
-			return 0, ErrInvalidLength
-		}
-		if depth == 0 {
-			return iNdEx, nil
-		}
-	}
-	return 0, io.ErrUnexpectedEOF
-}
-
-var (
-	ErrInvalidLength        = fmt.Errorf("proto: negative length found during unmarshaling")
-	ErrIntOverflow          = fmt.Errorf("proto: integer overflow")
-	ErrUnexpectedEndOfGroup = fmt.Errorf("proto: unexpected end of group")
-)

mk-rel.sh

@@ -3,7 +3,7 @@
 Z=`basename $0`
 die() {
     echo "$Z: $@" 1>&2
-    exit 0
+    exit 1
 }
 
 warn() {
@@ -18,6 +18,8 @@ case $BASH_VERSION in
 esac
 
 Rel=$PWD/releases
+Bindir=$Rel/bin
+mkdir -p $Bindir || die "can't make $Bindir"
 
 pkgit() {
     local os=$1
@@ -25,28 +27,26 @@ pkgit() {
     local rev=$3
     local arch="$os-$cpu"
     local tgz="$Rel/sigtool-${rev}_${arch}.tar.gz"
-    local bindir=./bin/$arch
+    local bindir=$Bindir/$arch
     local bin=sigtool
 
     if [ "$os" = "windows" ]; then
         bin=${bin}.exe
     fi
 
-    ./build -V $rev -s -a $arch || die "can't build $arch"
+    ./build -V $rev -b $Bindir -s -a $arch || die "can't build $arch"
     (cd $bindir && tar cf - $bin)  | gzip -9 > $tgz || die "can't tar $tgz"
 }
 
 xrev=$(git describe --always --dirty --abbrev=12) || exit 1
 if echo $xrev | grep -q dirty; then
-    #die "won't build releases; repo dirty!"
+    die "won't build releases; repo dirty!"
     true
 fi
 
 os="linux windows openbsd darwin"
 arch="amd64 arm64"
 
-mkdir -p $Rel
-
 for xx in $os; do
     for yy in $arch; do
         pkgit $xx $yy $xrev