1
0
Fork 0
forked from forgejo/forgejo

Integrate public as bindata optionally (#293)

* Dropped unused codekit config

* Integrated dynamic and static bindata for public

* Ignore public bindata

* Add a general generate make task

* Integrated flexible public assets into web command

* Updated vendoring, added all missiong govendor deps

* Made the linter happy with the bindata and dynamic code

* Moved public bindata definition to modules directory

* Ignoring the new bindata path now

* Updated to the new public modules import path

* Updated public bindata command and drop the new prefix
This commit is contained in:
Thomas Boerger 2016-11-29 17:26:36 +01:00 committed by Lunny Xiao
parent 4680c349dd
commit b6a95a8cb3
691 changed files with 305318 additions and 1272 deletions

22
vendor/github.com/pingcap/go-themis/LICENSE generated vendored Normal file
View file

@ -0,0 +1,22 @@
The MIT License (MIT)
Copyright (c) 2015 PingCAP
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in all
copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
SOFTWARE.

30
vendor/github.com/pingcap/go-themis/README.md generated vendored Normal file
View file

@ -0,0 +1,30 @@
# go-themis
[![Build Status](https://travis-ci.org/pingcap/go-themis.svg?branch=master)](https://travis-ci.org/pingcap/go-themis)
go-themis is a Go client for [pingcap/themis](https://github.com/pingcap/themis).
Themis provides cross-row/cross-table transaction on HBase based on [google's Percolator](http://research.google.com/pubs/pub36726.html).
go-themis is depends on [pingcap/go-hbase](https://github.com/pingcap/go-hbase).
Install:
```
go get -u github.com/pingcap/go-themis
```
Example:
```
tx := themis.NewTxn(c, oracles.NewLocalOracle())
put := hbase.NewPut([]byte("Row1"))
put.AddValue([]byte("cf"), []byte("q"), []byte("value"))
put2 := hbase.NewPut([]byte("Row2"))
put2.AddValue([]byte("cf"), []byte("q"), []byte("value"))
tx.Put(tblName, put)
tx.Put(tblName, put2)
tx.Commit()
```

547
vendor/github.com/pingcap/go-themis/Themis.pb.go generated vendored Normal file
View file

@ -0,0 +1,547 @@
// Code generated by protoc-gen-go.
// source: Themis.proto
// DO NOT EDIT!
/*
Package Themis is a generated protocol buffer package.
It is generated from these files:
Themis.proto
It has these top-level messages:
ThemisGetRequest
ThemisBatchGetRequest
ThemisBatchGetResponse
ThemisPrewrite
ThemisPrewriteRequest
ThemisPrewriteResponse
ThemisBatchPrewriteSecondaryRequest
ThemisBatchPrewriteSecondaryResponse
ThemisPrewriteResult
ThemisCommitRequest
ThemisCommitResponse
ThemisBatchCommitSecondaryRequest
ThemisBatchCommitSecondaryResponse
ThemisBatchCommitSecondaryResult
ThemisCommit
EraseLockRequest
EraseLockResponse
LockExpiredRequest
LockExpiredResponse
*/
package themis
import proto "github.com/golang/protobuf/proto"
import fmt "fmt"
import math "math"
import Client "github.com/pingcap/go-hbase/proto"
import Cell "github.com/pingcap/go-hbase/proto"
// Reference imports to suppress errors if they are not otherwise used.
var _ = proto.Marshal
var _ = fmt.Errorf
var _ = math.Inf
type ThemisGetRequest struct {
Get *Client.Get `protobuf:"bytes,1,req,name=get" json:"get,omitempty"`
StartTs *uint64 `protobuf:"varint,2,req,name=startTs" json:"startTs,omitempty"`
IgnoreLock *bool `protobuf:"varint,3,req,name=ignoreLock" json:"ignoreLock,omitempty"`
XXX_unrecognized []byte `json:"-"`
}
func (m *ThemisGetRequest) Reset() { *m = ThemisGetRequest{} }
func (m *ThemisGetRequest) String() string { return proto.CompactTextString(m) }
func (*ThemisGetRequest) ProtoMessage() {}
func (m *ThemisGetRequest) GetGet() *Client.Get {
if m != nil {
return m.Get
}
return nil
}
func (m *ThemisGetRequest) GetStartTs() uint64 {
if m != nil && m.StartTs != nil {
return *m.StartTs
}
return 0
}
func (m *ThemisGetRequest) GetIgnoreLock() bool {
if m != nil && m.IgnoreLock != nil {
return *m.IgnoreLock
}
return false
}
type ThemisBatchGetRequest struct {
Gets []*Client.Get `protobuf:"bytes,1,rep,name=gets" json:"gets,omitempty"`
StartTs *uint64 `protobuf:"varint,2,req,name=startTs" json:"startTs,omitempty"`
IgnoreLock *bool `protobuf:"varint,3,req,name=ignoreLock" json:"ignoreLock,omitempty"`
XXX_unrecognized []byte `json:"-"`
}
func (m *ThemisBatchGetRequest) Reset() { *m = ThemisBatchGetRequest{} }
func (m *ThemisBatchGetRequest) String() string { return proto.CompactTextString(m) }
func (*ThemisBatchGetRequest) ProtoMessage() {}
func (m *ThemisBatchGetRequest) GetGets() []*Client.Get {
if m != nil {
return m.Gets
}
return nil
}
func (m *ThemisBatchGetRequest) GetStartTs() uint64 {
if m != nil && m.StartTs != nil {
return *m.StartTs
}
return 0
}
func (m *ThemisBatchGetRequest) GetIgnoreLock() bool {
if m != nil && m.IgnoreLock != nil {
return *m.IgnoreLock
}
return false
}
type ThemisBatchGetResponse struct {
Rs []*Client.Result `protobuf:"bytes,1,rep,name=rs" json:"rs,omitempty"`
XXX_unrecognized []byte `json:"-"`
}
func (m *ThemisBatchGetResponse) Reset() { *m = ThemisBatchGetResponse{} }
func (m *ThemisBatchGetResponse) String() string { return proto.CompactTextString(m) }
func (*ThemisBatchGetResponse) ProtoMessage() {}
func (m *ThemisBatchGetResponse) GetRs() []*Client.Result {
if m != nil {
return m.Rs
}
return nil
}
type ThemisPrewrite struct {
Row []byte `protobuf:"bytes,1,req,name=row" json:"row,omitempty"`
Mutations []*Cell.Cell `protobuf:"bytes,2,rep,name=mutations" json:"mutations,omitempty"`
XXX_unrecognized []byte `json:"-"`
}
func (m *ThemisPrewrite) Reset() { *m = ThemisPrewrite{} }
func (m *ThemisPrewrite) String() string { return proto.CompactTextString(m) }
func (*ThemisPrewrite) ProtoMessage() {}
func (m *ThemisPrewrite) GetRow() []byte {
if m != nil {
return m.Row
}
return nil
}
func (m *ThemisPrewrite) GetMutations() []*Cell.Cell {
if m != nil {
return m.Mutations
}
return nil
}
type ThemisPrewriteRequest struct {
ThemisPrewrite *ThemisPrewrite `protobuf:"bytes,1,req,name=themisPrewrite" json:"themisPrewrite,omitempty"`
PrewriteTs *uint64 `protobuf:"varint,2,req,name=prewriteTs" json:"prewriteTs,omitempty"`
SecondaryLock []byte `protobuf:"bytes,3,req,name=secondaryLock" json:"secondaryLock,omitempty"`
PrimaryLock []byte `protobuf:"bytes,4,req,name=primaryLock" json:"primaryLock,omitempty"`
PrimaryIndex *int32 `protobuf:"varint,5,req,name=primaryIndex" json:"primaryIndex,omitempty"`
XXX_unrecognized []byte `json:"-"`
}
func (m *ThemisPrewriteRequest) Reset() { *m = ThemisPrewriteRequest{} }
func (m *ThemisPrewriteRequest) String() string { return proto.CompactTextString(m) }
func (*ThemisPrewriteRequest) ProtoMessage() {}
func (m *ThemisPrewriteRequest) GetThemisPrewrite() *ThemisPrewrite {
if m != nil {
return m.ThemisPrewrite
}
return nil
}
func (m *ThemisPrewriteRequest) GetPrewriteTs() uint64 {
if m != nil && m.PrewriteTs != nil {
return *m.PrewriteTs
}
return 0
}
func (m *ThemisPrewriteRequest) GetSecondaryLock() []byte {
if m != nil {
return m.SecondaryLock
}
return nil
}
func (m *ThemisPrewriteRequest) GetPrimaryLock() []byte {
if m != nil {
return m.PrimaryLock
}
return nil
}
func (m *ThemisPrewriteRequest) GetPrimaryIndex() int32 {
if m != nil && m.PrimaryIndex != nil {
return *m.PrimaryIndex
}
return 0
}
type ThemisPrewriteResponse struct {
ThemisPrewriteResult *ThemisPrewriteResult `protobuf:"bytes,1,opt,name=themisPrewriteResult" json:"themisPrewriteResult,omitempty"`
XXX_unrecognized []byte `json:"-"`
}
func (m *ThemisPrewriteResponse) Reset() { *m = ThemisPrewriteResponse{} }
func (m *ThemisPrewriteResponse) String() string { return proto.CompactTextString(m) }
func (*ThemisPrewriteResponse) ProtoMessage() {}
func (m *ThemisPrewriteResponse) GetThemisPrewriteResult() *ThemisPrewriteResult {
if m != nil {
return m.ThemisPrewriteResult
}
return nil
}
type ThemisBatchPrewriteSecondaryRequest struct {
ThemisPrewrite []*ThemisPrewrite `protobuf:"bytes,1,rep,name=themisPrewrite" json:"themisPrewrite,omitempty"`
PrewriteTs *uint64 `protobuf:"varint,2,req,name=prewriteTs" json:"prewriteTs,omitempty"`
SecondaryLock []byte `protobuf:"bytes,3,req,name=secondaryLock" json:"secondaryLock,omitempty"`
XXX_unrecognized []byte `json:"-"`
}
func (m *ThemisBatchPrewriteSecondaryRequest) Reset() { *m = ThemisBatchPrewriteSecondaryRequest{} }
func (m *ThemisBatchPrewriteSecondaryRequest) String() string { return proto.CompactTextString(m) }
func (*ThemisBatchPrewriteSecondaryRequest) ProtoMessage() {}
func (m *ThemisBatchPrewriteSecondaryRequest) GetThemisPrewrite() []*ThemisPrewrite {
if m != nil {
return m.ThemisPrewrite
}
return nil
}
func (m *ThemisBatchPrewriteSecondaryRequest) GetPrewriteTs() uint64 {
if m != nil && m.PrewriteTs != nil {
return *m.PrewriteTs
}
return 0
}
func (m *ThemisBatchPrewriteSecondaryRequest) GetSecondaryLock() []byte {
if m != nil {
return m.SecondaryLock
}
return nil
}
type ThemisBatchPrewriteSecondaryResponse struct {
ThemisPrewriteResult []*ThemisPrewriteResult `protobuf:"bytes,1,rep,name=themisPrewriteResult" json:"themisPrewriteResult,omitempty"`
RowsNotInRegion [][]byte `protobuf:"bytes,2,rep,name=rowsNotInRegion" json:"rowsNotInRegion,omitempty"`
XXX_unrecognized []byte `json:"-"`
}
func (m *ThemisBatchPrewriteSecondaryResponse) Reset() { *m = ThemisBatchPrewriteSecondaryResponse{} }
func (m *ThemisBatchPrewriteSecondaryResponse) String() string { return proto.CompactTextString(m) }
func (*ThemisBatchPrewriteSecondaryResponse) ProtoMessage() {}
func (m *ThemisBatchPrewriteSecondaryResponse) GetThemisPrewriteResult() []*ThemisPrewriteResult {
if m != nil {
return m.ThemisPrewriteResult
}
return nil
}
func (m *ThemisBatchPrewriteSecondaryResponse) GetRowsNotInRegion() [][]byte {
if m != nil {
return m.RowsNotInRegion
}
return nil
}
type ThemisPrewriteResult struct {
NewerWriteTs *int64 `protobuf:"varint,1,req,name=newerWriteTs" json:"newerWriteTs,omitempty"`
ExistLock []byte `protobuf:"bytes,2,req,name=existLock" json:"existLock,omitempty"`
Family []byte `protobuf:"bytes,3,req,name=family" json:"family,omitempty"`
Qualifier []byte `protobuf:"bytes,4,req,name=qualifier" json:"qualifier,omitempty"`
LockExpired *bool `protobuf:"varint,5,req,name=lockExpired" json:"lockExpired,omitempty"`
Row []byte `protobuf:"bytes,6,req,name=row" json:"row,omitempty"`
XXX_unrecognized []byte `json:"-"`
}
func (m *ThemisPrewriteResult) Reset() { *m = ThemisPrewriteResult{} }
func (m *ThemisPrewriteResult) String() string { return proto.CompactTextString(m) }
func (*ThemisPrewriteResult) ProtoMessage() {}
func (m *ThemisPrewriteResult) GetNewerWriteTs() int64 {
if m != nil && m.NewerWriteTs != nil {
return *m.NewerWriteTs
}
return 0
}
func (m *ThemisPrewriteResult) GetExistLock() []byte {
if m != nil {
return m.ExistLock
}
return nil
}
func (m *ThemisPrewriteResult) GetFamily() []byte {
if m != nil {
return m.Family
}
return nil
}
func (m *ThemisPrewriteResult) GetQualifier() []byte {
if m != nil {
return m.Qualifier
}
return nil
}
func (m *ThemisPrewriteResult) GetLockExpired() bool {
if m != nil && m.LockExpired != nil {
return *m.LockExpired
}
return false
}
func (m *ThemisPrewriteResult) GetRow() []byte {
if m != nil {
return m.Row
}
return nil
}
type ThemisCommitRequest struct {
ThemisCommit *ThemisCommit `protobuf:"bytes,1,req,name=themisCommit" json:"themisCommit,omitempty"`
XXX_unrecognized []byte `json:"-"`
}
func (m *ThemisCommitRequest) Reset() { *m = ThemisCommitRequest{} }
func (m *ThemisCommitRequest) String() string { return proto.CompactTextString(m) }
func (*ThemisCommitRequest) ProtoMessage() {}
func (m *ThemisCommitRequest) GetThemisCommit() *ThemisCommit {
if m != nil {
return m.ThemisCommit
}
return nil
}
type ThemisCommitResponse struct {
Result *bool `protobuf:"varint,1,req,name=result" json:"result,omitempty"`
XXX_unrecognized []byte `json:"-"`
}
func (m *ThemisCommitResponse) Reset() { *m = ThemisCommitResponse{} }
func (m *ThemisCommitResponse) String() string { return proto.CompactTextString(m) }
func (*ThemisCommitResponse) ProtoMessage() {}
func (m *ThemisCommitResponse) GetResult() bool {
if m != nil && m.Result != nil {
return *m.Result
}
return false
}
type ThemisBatchCommitSecondaryRequest struct {
ThemisCommit []*ThemisCommit `protobuf:"bytes,1,rep,name=themisCommit" json:"themisCommit,omitempty"`
XXX_unrecognized []byte `json:"-"`
}
func (m *ThemisBatchCommitSecondaryRequest) Reset() { *m = ThemisBatchCommitSecondaryRequest{} }
func (m *ThemisBatchCommitSecondaryRequest) String() string { return proto.CompactTextString(m) }
func (*ThemisBatchCommitSecondaryRequest) ProtoMessage() {}
func (m *ThemisBatchCommitSecondaryRequest) GetThemisCommit() []*ThemisCommit {
if m != nil {
return m.ThemisCommit
}
return nil
}
type ThemisBatchCommitSecondaryResponse struct {
BatchCommitSecondaryResult []*ThemisBatchCommitSecondaryResult `protobuf:"bytes,1,rep,name=batchCommitSecondaryResult" json:"batchCommitSecondaryResult,omitempty"`
XXX_unrecognized []byte `json:"-"`
}
func (m *ThemisBatchCommitSecondaryResponse) Reset() { *m = ThemisBatchCommitSecondaryResponse{} }
func (m *ThemisBatchCommitSecondaryResponse) String() string { return proto.CompactTextString(m) }
func (*ThemisBatchCommitSecondaryResponse) ProtoMessage() {}
func (m *ThemisBatchCommitSecondaryResponse) GetBatchCommitSecondaryResult() []*ThemisBatchCommitSecondaryResult {
if m != nil {
return m.BatchCommitSecondaryResult
}
return nil
}
type ThemisBatchCommitSecondaryResult struct {
Row []byte `protobuf:"bytes,1,req,name=row" json:"row,omitempty"`
Success *bool `protobuf:"varint,2,req,name=success" json:"success,omitempty"`
XXX_unrecognized []byte `json:"-"`
}
func (m *ThemisBatchCommitSecondaryResult) Reset() { *m = ThemisBatchCommitSecondaryResult{} }
func (m *ThemisBatchCommitSecondaryResult) String() string { return proto.CompactTextString(m) }
func (*ThemisBatchCommitSecondaryResult) ProtoMessage() {}
func (m *ThemisBatchCommitSecondaryResult) GetRow() []byte {
if m != nil {
return m.Row
}
return nil
}
func (m *ThemisBatchCommitSecondaryResult) GetSuccess() bool {
if m != nil && m.Success != nil {
return *m.Success
}
return false
}
type ThemisCommit struct {
Row []byte `protobuf:"bytes,1,req,name=row" json:"row,omitempty"`
Mutations []*Cell.Cell `protobuf:"bytes,2,rep,name=mutations" json:"mutations,omitempty"`
PrewriteTs *uint64 `protobuf:"varint,3,req,name=prewriteTs" json:"prewriteTs,omitempty"`
CommitTs *uint64 `protobuf:"varint,4,req,name=commitTs" json:"commitTs,omitempty"`
PrimaryIndex *int32 `protobuf:"varint,5,req,name=primaryIndex" json:"primaryIndex,omitempty"`
XXX_unrecognized []byte `json:"-"`
}
func (m *ThemisCommit) Reset() { *m = ThemisCommit{} }
func (m *ThemisCommit) String() string { return proto.CompactTextString(m) }
func (*ThemisCommit) ProtoMessage() {}
func (m *ThemisCommit) GetRow() []byte {
if m != nil {
return m.Row
}
return nil
}
func (m *ThemisCommit) GetMutations() []*Cell.Cell {
if m != nil {
return m.Mutations
}
return nil
}
func (m *ThemisCommit) GetPrewriteTs() uint64 {
if m != nil && m.PrewriteTs != nil {
return *m.PrewriteTs
}
return 0
}
func (m *ThemisCommit) GetCommitTs() uint64 {
if m != nil && m.CommitTs != nil {
return *m.CommitTs
}
return 0
}
func (m *ThemisCommit) GetPrimaryIndex() int32 {
if m != nil && m.PrimaryIndex != nil {
return *m.PrimaryIndex
}
return 0
}
type EraseLockRequest struct {
Row []byte `protobuf:"bytes,1,req,name=row" json:"row,omitempty"`
Family []byte `protobuf:"bytes,2,req,name=family" json:"family,omitempty"`
Qualifier []byte `protobuf:"bytes,3,req,name=qualifier" json:"qualifier,omitempty"`
PrewriteTs *uint64 `protobuf:"varint,4,req,name=prewriteTs" json:"prewriteTs,omitempty"`
XXX_unrecognized []byte `json:"-"`
}
func (m *EraseLockRequest) Reset() { *m = EraseLockRequest{} }
func (m *EraseLockRequest) String() string { return proto.CompactTextString(m) }
func (*EraseLockRequest) ProtoMessage() {}
func (m *EraseLockRequest) GetRow() []byte {
if m != nil {
return m.Row
}
return nil
}
func (m *EraseLockRequest) GetFamily() []byte {
if m != nil {
return m.Family
}
return nil
}
func (m *EraseLockRequest) GetQualifier() []byte {
if m != nil {
return m.Qualifier
}
return nil
}
func (m *EraseLockRequest) GetPrewriteTs() uint64 {
if m != nil && m.PrewriteTs != nil {
return *m.PrewriteTs
}
return 0
}
type EraseLockResponse struct {
Lock []byte `protobuf:"bytes,1,opt,name=lock" json:"lock,omitempty"`
XXX_unrecognized []byte `json:"-"`
}
func (m *EraseLockResponse) Reset() { *m = EraseLockResponse{} }
func (m *EraseLockResponse) String() string { return proto.CompactTextString(m) }
func (*EraseLockResponse) ProtoMessage() {}
func (m *EraseLockResponse) GetLock() []byte {
if m != nil {
return m.Lock
}
return nil
}
type LockExpiredRequest struct {
Timestamp *uint64 `protobuf:"varint,1,req,name=timestamp" json:"timestamp,omitempty"`
XXX_unrecognized []byte `json:"-"`
}
func (m *LockExpiredRequest) Reset() { *m = LockExpiredRequest{} }
func (m *LockExpiredRequest) String() string { return proto.CompactTextString(m) }
func (*LockExpiredRequest) ProtoMessage() {}
func (m *LockExpiredRequest) GetTimestamp() uint64 {
if m != nil && m.Timestamp != nil {
return *m.Timestamp
}
return 0
}
type LockExpiredResponse struct {
Expired *bool `protobuf:"varint,1,req,name=expired" json:"expired,omitempty"`
XXX_unrecognized []byte `json:"-"`
}
func (m *LockExpiredResponse) Reset() { *m = LockExpiredResponse{} }
func (m *LockExpiredResponse) String() string { return proto.CompactTextString(m) }
func (*LockExpiredResponse) ProtoMessage() {}
func (m *LockExpiredResponse) GetExpired() bool {
if m != nil && m.Expired != nil {
return *m.Expired
}
return false
}

20
vendor/github.com/pingcap/go-themis/consts.go generated vendored Normal file
View file

@ -0,0 +1,20 @@
package themis
import "strings"
var (
PutFamilyName = []byte("#p")
DelFamilyName = []byte("#d")
LockFamilyName = []byte("L")
)
const (
ThemisServiceName string = "ThemisService"
)
func isWrongRegionErr(err error) bool {
if err != nil {
return strings.Contains(err.Error(), "org.apache.hadoop.hbase.regionserver.WrongRegionException")
}
return false
}

58
vendor/github.com/pingcap/go-themis/lock.go generated vendored Normal file
View file

@ -0,0 +1,58 @@
package themis
import "github.com/pingcap/go-hbase"
// LockRole is the role of lock
type LockRole int
func (l LockRole) String() string {
if l == RolePrimary {
return "primary"
}
return "secondary"
}
const (
// RolePrimary means this row is primary
RolePrimary LockRole = iota
// RoleSecondary means this row is secondary
RoleSecondary
)
type Lock interface {
// SetCoordinate sets lock's coordinate
SetCoordinate(c *hbase.ColumnCoordinate)
// Coordinate returns the lock's coordinate
Coordinate() *hbase.ColumnCoordinate
// Timestamp returns startTs of the transction which owned this lock
Timestamp() uint64
// SetExpired sets the lock's expired status.
SetExpired(b bool)
// IsExpired returns if lock is expired.
IsExpired() bool
// Type returns the lock's type, Put or Delete
Type() hbase.Type
// Role returns LockRole, primary or secondary
Role() LockRole
// not used now
Context() interface{}
// valid only Role == Primary
Secondaries() []Lock
// Primary returns the primary lock of this lock
Primary() Lock
// Encode encodes the lock to byte slice
Encode() []byte
}
type LockManager interface {
// CleanLock if clean lock success, first return value is transction's commit
// timestamp, otherwise, the second return value is transction's primary
// lock.
CleanLock(c *hbase.ColumnCoordinate, prewriteTs uint64) (uint64, Lock, error)
// EraseLockAndData removes lock and data.
EraseLockAndData(c *hbase.ColumnCoordinate, prewriteTs uint64) error
// GetCommitTimestamp returns a committed transction's commit timestamp.
GetCommitTimestamp(c *hbase.ColumnCoordinate, prewriteTs uint64) (uint64, error)
// [startTs, endTs]
IsLockExists(c *hbase.ColumnCoordinate, startTs, endTs uint64) (bool, error)
}

233
vendor/github.com/pingcap/go-themis/mutation_cache.go generated vendored Normal file
View file

@ -0,0 +1,233 @@
package themis
import (
"fmt"
"sort"
"github.com/juju/errors"
"github.com/ngaut/log"
"github.com/pingcap/go-hbase"
"github.com/pingcap/go-hbase/proto"
)
type mutationValuePair struct {
typ hbase.Type
value []byte
}
func (mp *mutationValuePair) String() string {
return fmt.Sprintf("type: %d value: %s", mp.typ, mp.value)
}
type columnMutation struct {
*hbase.Column
*mutationValuePair
}
func getEntriesFromDel(p *hbase.Delete) ([]*columnMutation, error) {
errMsg := "must set at least one column for themis delete"
if len(p.FamilyQuals) == 0 {
return nil, errors.New(errMsg)
}
var ret []*columnMutation
for f, _ := range p.Families {
quilifiers := p.FamilyQuals[f]
if len(quilifiers) == 0 {
return nil, errors.New(errMsg)
}
for q, _ := range quilifiers {
mutation := &columnMutation{
Column: &hbase.Column{
Family: []byte(f),
Qual: []byte(q),
},
mutationValuePair: &mutationValuePair{
typ: hbase.TypeDeleteColumn,
},
}
ret = append(ret, mutation)
}
}
return ret, nil
}
func getEntriesFromPut(p *hbase.Put) []*columnMutation {
var ret []*columnMutation
for i, f := range p.Families {
qualifiers := p.Qualifiers[i]
for j, q := range qualifiers {
mutation := &columnMutation{
Column: &hbase.Column{
Family: f,
Qual: q,
},
mutationValuePair: &mutationValuePair{
typ: hbase.TypePut,
value: p.Values[i][j],
},
}
ret = append(ret, mutation)
}
}
return ret
}
func (cm *columnMutation) toCell() *proto.Cell {
ret := &proto.Cell{
Family: cm.Family,
Qualifier: cm.Qual,
Value: cm.value,
}
if cm.typ == hbase.TypePut { // put
ret.CellType = proto.CellType_PUT.Enum()
} else if cm.typ == hbase.TypeMinimum { // onlyLock
ret.CellType = proto.CellType_MINIMUM.Enum()
} else { // delete, themis delete API only support delete column
ret.CellType = proto.CellType_DELETE_COLUMN.Enum()
}
return ret
}
type rowMutation struct {
tbl []byte
row []byte
// mutations := { 'cf:col' => mutationValuePair }
mutations map[string]*mutationValuePair
}
func (r *rowMutation) getColumns() []hbase.Column {
var ret []hbase.Column
for k, _ := range r.mutations {
c := &hbase.Column{}
// TODO: handle error, now just ignore
if err := c.ParseFromString(k); err != nil {
log.Warnf("parse from string error, column: %s, mutation: %s, error: %v", c, k, err)
}
ret = append(ret, *c)
}
return ret
}
func (r *rowMutation) getSize() int {
return len(r.mutations)
}
func (r *rowMutation) getType(c hbase.Column) hbase.Type {
p, ok := r.mutations[c.String()]
if !ok {
return hbase.TypeMinimum
}
return p.typ
}
func newRowMutation(tbl, row []byte) *rowMutation {
return &rowMutation{
tbl: tbl,
row: row,
mutations: map[string]*mutationValuePair{},
}
}
func (r *rowMutation) addMutation(c *hbase.Column, typ hbase.Type, val []byte, onlyLock bool) {
// 3 scene: put, delete, onlyLock
// if it is onlyLock scene, then has not data modify, when has contained the qualifier, can't replace exist value,
// becuase put or delete operation has add mutation
if onlyLock && r.mutations[c.String()] != nil {
return
}
r.mutations[c.String()] = &mutationValuePair{
typ: typ,
value: val,
}
}
func (r *rowMutation) mutationList(withValue bool) []*columnMutation {
var ret []*columnMutation
var keys []string
for k, _ := range r.mutations {
keys = append(keys, k)
}
sort.Strings(keys)
for _, k := range keys {
v := &mutationValuePair{
typ: r.mutations[k].typ,
}
if withValue {
v.value = r.mutations[k].value
}
c := &hbase.Column{}
// TODO: handle error, now just ignore
if err := c.ParseFromString(k); err != nil {
log.Warnf("parse from string error, column: %s, mutation: %s, error: %v", c, k, err)
}
ret = append(ret, &columnMutation{
Column: c,
mutationValuePair: v,
})
}
return ret
}
type columnMutationCache struct {
// mutations => {table => { rowKey => row mutations } }
mutations map[string]map[string]*rowMutation
}
func newColumnMutationCache() *columnMutationCache {
return &columnMutationCache{
mutations: map[string]map[string]*rowMutation{},
}
}
func (c *columnMutationCache) addMutation(tbl []byte, row []byte, col *hbase.Column, t hbase.Type, v []byte, onlyLock bool) {
tblRowMutations, ok := c.mutations[string(tbl)]
if !ok {
// create table mutation map
tblRowMutations = map[string]*rowMutation{}
c.mutations[string(tbl)] = tblRowMutations
}
rowMutations, ok := tblRowMutations[string(row)]
if !ok {
// create row mutation map
rowMutations = newRowMutation(tbl, row)
tblRowMutations[string(row)] = rowMutations
}
rowMutations.addMutation(col, t, v, onlyLock)
}
func (c *columnMutationCache) getMutation(cc *hbase.ColumnCoordinate) *mutationValuePair {
t, ok := c.mutations[string(cc.Table)]
if !ok {
return nil
}
rowMutation, ok := t[string(cc.Row)]
if !ok {
return nil
}
p, ok := rowMutation.mutations[cc.GetColumn().String()]
if !ok {
return nil
}
return p
}
func (c *columnMutationCache) getRowCount() int {
ret := 0
for _, v := range c.mutations {
ret += len(v)
}
return ret
}
func (c *columnMutationCache) getMutationCount() int {
ret := 0
for _, v := range c.mutations {
for _, vv := range v {
ret += len(vv.mutationList(false))
}
}
return ret
}

6
vendor/github.com/pingcap/go-themis/oracle/oracle.go generated vendored Normal file
View file

@ -0,0 +1,6 @@
package oracle
type Oracle interface {
GetTimestamp() (uint64, error)
IsExpired(lockTimestamp uint64, TTL uint64) bool
}

View file

@ -0,0 +1,42 @@
package oracles
import (
"sync"
"time"
"github.com/pingcap/go-themis/oracle"
)
const epochShiftBits = 18
var _ oracle.Oracle = &localOracle{}
type localOracle struct {
mu sync.Mutex
lastTimeStampTs int64
n int64
}
// NewLocalOracle creates an Oracle that use local time as data source.
func NewLocalOracle() oracle.Oracle {
return &localOracle{}
}
func (l *localOracle) IsExpired(lockTs uint64, TTL uint64) bool {
beginMs := lockTs >> epochShiftBits
return uint64(time.Now().UnixNano()/int64(time.Millisecond)) >= (beginMs + TTL)
}
func (l *localOracle) GetTimestamp() (uint64, error) {
l.mu.Lock()
defer l.mu.Unlock()
ts := (time.Now().UnixNano() / int64(time.Millisecond)) << epochShiftBits
if l.lastTimeStampTs == ts {
l.n++
return uint64(ts + l.n), nil
} else {
l.lastTimeStampTs = ts
l.n = 0
}
return uint64(ts), nil
}

View file

@ -0,0 +1,48 @@
package oracles
import (
"time"
"github.com/juju/errors"
"github.com/ngaut/tso/client"
"github.com/pingcap/go-themis/oracle"
)
const maxRetryCnt = 3
var _ oracle.Oracle = &remoteOracle{}
// remoteOracle is an oracle that use a remote data source.
type remoteOracle struct {
c *client.Client
}
// NewRemoteOracle creates an oracle that use a remote data source.
// Refer https://github.com/ngaut/tso for more details.
func NewRemoteOracle(zks, path string) oracle.Oracle {
return &remoteOracle{
c: client.NewClient(&client.Conf{
ZKAddr: zks,
RootPath: path,
}),
}
}
func (t *remoteOracle) IsExpired(lockTs uint64, TTL uint64) bool {
beginMs := lockTs >> epochShiftBits
// TODO records the local wall time when getting beginMs from TSO
return uint64(time.Now().UnixNano()/int64(time.Millisecond)) >= (beginMs + TTL)
}
// GetTimestamp gets timestamp from remote data source.
func (t *remoteOracle) GetTimestamp() (uint64, error) {
var err error
for i := 0; i < maxRetryCnt; i++ {
ts, e := t.c.GoGetTimestamp().GetTS()
if e == nil {
return uint64((ts.Physical << epochShiftBits) + ts.Logical), nil
}
err = errors.Trace(e)
}
return 0, err
}

133
vendor/github.com/pingcap/go-themis/themis_lock.go generated vendored Normal file
View file

@ -0,0 +1,133 @@
package themis
import (
"bytes"
"encoding/binary"
"io"
"github.com/juju/errors"
"github.com/pingcap/go-hbase"
"github.com/pingcap/go-hbase/iohelper"
)
var (
_ Lock = (*themisPrimaryLock)(nil)
_ Lock = (*themisSecondaryLock)(nil)
)
type themisLock struct {
// lock coordinate, table, row, cf, q
coordinate *hbase.ColumnCoordinate
// lock type: put/delete/minimal(lock only)
typ hbase.Type
// prewrite ts
ts uint64
// not used, for alignment
wallTs uint64
// not used, for alignment
clientAddr string
expired bool
}
func (l *themisLock) Timestamp() uint64 {
return l.ts
}
func (l *themisLock) IsExpired() bool {
return l.expired
}
func (l *themisLock) SetExpired(b bool) {
l.expired = b
}
func (l *themisLock) SetCoordinate(c *hbase.ColumnCoordinate) {
l.coordinate = c
}
func (l *themisLock) Coordinate() *hbase.ColumnCoordinate {
return l.coordinate
}
func (l *themisLock) Context() interface{} {
return nil
}
func (l *themisLock) Type() hbase.Type {
return l.typ
}
func (l *themisLock) write(w io.Writer) {
binary.Write(w, binary.BigEndian, byte(l.typ))
binary.Write(w, binary.BigEndian, int64(l.ts))
// write client addr
iohelper.WriteVarBytes(w, []byte(l.clientAddr))
binary.Write(w, binary.BigEndian, int64(l.wallTs))
}
func (l *themisLock) parse(r iohelper.ByteMultiReader) error {
// read type
var typ uint8
err := binary.Read(r, binary.BigEndian, &typ)
if err != nil {
return errors.Trace(err)
}
l.typ = hbase.Type(typ)
// read ts
var ts int64
err = binary.Read(r, binary.BigEndian, &ts)
if err != nil {
return errors.Trace(err)
}
l.ts = uint64(ts)
// read client addr
sz, err := binary.ReadUvarint(r)
if err != nil {
return errors.Trace(err)
}
addr := make([]byte, sz)
r.Read(addr)
l.clientAddr = string(addr)
// read wall time
var wallTs int64
err = binary.Read(r, binary.BigEndian, &wallTs)
if err != nil {
return errors.Trace(err)
}
l.wallTs = uint64(wallTs)
return nil
}
func parseLockFromBytes(b []byte) (Lock, error) {
buf := bytes.NewBuffer(b)
var isPrimary uint8
err := binary.Read(buf, binary.BigEndian, &isPrimary)
if err != nil {
return nil, errors.Trace(err)
}
var ret Lock
if isPrimary == 1 {
l := newThemisPrimaryLock()
err = l.parse(buf)
ret = l
} else {
l := newThemisSecondaryLock()
err = l.parse(buf)
ret = l
}
if err != nil {
return nil, errors.Trace(err)
}
return ret, nil
}
func isLockResult(r *hbase.ResultRow) bool {
return len(r.SortedColumns) > 0 && isLockColumn(r.SortedColumns[0].Column)
}
func isLockColumn(c hbase.Column) bool {
return bytes.Compare(c.Family, LockFamilyName) == 0
}

View file

@ -0,0 +1,147 @@
package themis
import (
"bytes"
"encoding/binary"
"math"
"strings"
"github.com/juju/errors"
"github.com/ngaut/log"
"github.com/pingcap/go-hbase"
)
var _ LockManager = (*themisLockManager)(nil)
type themisLockManager struct {
rpc *themisRPC
hbaseClient hbase.HBaseClient
}
func newThemisLockManager(rpc *themisRPC, hbaseCli hbase.HBaseClient) LockManager {
return &themisLockManager{
rpc: rpc,
hbaseClient: hbaseCli,
}
}
func getDataColFromMetaCol(lockOrWriteCol hbase.Column) hbase.Column {
// get data column from lock column
// key is like => L:family#qual, #p:family#qual
parts := strings.Split(string(lockOrWriteCol.Qual), "#")
if len(parts) != 2 {
return lockOrWriteCol
}
c := hbase.Column{
Family: []byte(parts[0]),
Qual: []byte(parts[1]),
}
return c
}
func getLocksFromResults(tbl []byte, lockKvs []*hbase.Kv, client *themisRPC) ([]Lock, error) {
var locks []Lock
for _, kv := range lockKvs {
col := &hbase.ColumnCoordinate{
Table: tbl,
Row: kv.Row,
Column: hbase.Column{
Family: kv.Family,
Qual: kv.Qual,
},
}
if !isLockColumn(col.Column) {
return nil, errors.New("invalid lock")
}
l, err := parseLockFromBytes(kv.Value)
if err != nil {
return nil, errors.Trace(err)
}
cc := &hbase.ColumnCoordinate{
Table: tbl,
Row: kv.Row,
Column: getDataColFromMetaCol(col.Column),
}
l.SetCoordinate(cc)
client.checkAndSetLockIsExpired(l)
locks = append(locks, l)
}
return locks, nil
}
func (m *themisLockManager) IsLockExists(cc *hbase.ColumnCoordinate, startTs, endTs uint64) (bool, error) {
get := hbase.NewGet(cc.Row)
get.AddTimeRange(startTs, endTs+1)
get.AddStringColumn(string(LockFamilyName), string(cc.Family)+"#"+string(cc.Qual))
// check if lock exists
rs, err := m.hbaseClient.Get(string(cc.Table), get)
if err != nil {
return false, errors.Trace(err)
}
// primary lock has been released
if rs == nil {
return false, nil
}
return true, nil
}
func (m *themisLockManager) GetCommitTimestamp(cc *hbase.ColumnCoordinate, prewriteTs uint64) (uint64, error) {
g := hbase.NewGet(cc.Row)
// add put write column
qual := string(cc.Family) + "#" + string(cc.Qual)
g.AddStringColumn("#p", qual)
// add del write column
g.AddStringColumn("#d", qual)
// time range => [ours startTs, +Inf)
g.AddTimeRange(prewriteTs, math.MaxInt64)
g.SetMaxVersion(math.MaxInt32)
r, err := m.hbaseClient.Get(string(cc.Table), g)
if err != nil {
return 0, errors.Trace(err)
}
// may delete by other client
if r == nil {
return 0, nil
}
for _, kv := range r.SortedColumns {
for commitTs, val := range kv.Values {
var ts uint64
binary.Read(bytes.NewBuffer(val), binary.BigEndian, &ts)
if ts == prewriteTs {
// get this commit's commitTs
return commitTs, nil
}
}
}
// no such transction
return 0, nil
}
func (m *themisLockManager) CleanLock(cc *hbase.ColumnCoordinate, prewriteTs uint64) (uint64, Lock, error) {
l, err := m.rpc.getLockAndErase(cc, prewriteTs)
if err != nil {
return 0, nil, errors.Trace(err)
}
pl, _ := l.(*themisPrimaryLock)
// if primary lock is nil, means someothers have already committed
if pl == nil {
commitTs, err := m.GetCommitTimestamp(cc, prewriteTs)
if err != nil {
return 0, nil, errors.Trace(err)
}
return commitTs, nil, nil
}
return 0, pl, nil
}
func (m *themisLockManager) EraseLockAndData(cc *hbase.ColumnCoordinate, prewriteTs uint64) error {
log.Debugf("erase row=%q txn=%d", cc.Row, prewriteTs)
d := hbase.NewDelete(cc.Row)
d.AddColumnWithTimestamp(LockFamilyName, []byte(string(cc.Family)+"#"+string(cc.Qual)), prewriteTs)
d.AddColumnWithTimestamp(cc.Family, cc.Qual, prewriteTs)
ok, err := m.hbaseClient.Delete(string(cc.Table), d)
if !ok {
log.Error(err)
}
return errors.Trace(err)
}

View file

@ -0,0 +1,112 @@
package themis
import (
"bytes"
"encoding/binary"
"github.com/juju/errors"
"github.com/ngaut/log"
"github.com/pingcap/go-hbase"
"github.com/pingcap/go-hbase/iohelper"
)
type themisPrimaryLock struct {
*themisLock
// {coordinate => type}
secondaries map[string]hbase.Type
}
func newThemisPrimaryLock() *themisPrimaryLock {
return &themisPrimaryLock{
themisLock: &themisLock{
clientAddr: "null",
},
secondaries: map[string]hbase.Type{},
}
}
func (l *themisPrimaryLock) Primary() Lock {
return l
}
func (l *themisPrimaryLock) Secondaries() []Lock {
var slocks []Lock
for k, v := range l.secondaries {
c := &hbase.ColumnCoordinate{}
// TODO: handle error, now just ignore
if err := c.ParseFromString(k); err != nil {
log.Warnf("parse from string error, column coordinate: %s, secondary: %s, error: %v", c, k, err)
continue
}
slock := newThemisSecondaryLock()
slock.primaryCoordinate = l.coordinate
slock.coordinate = c
slock.ts = l.ts
slock.typ = v
slocks = append(slocks, slock)
}
return slocks
}
func (l *themisPrimaryLock) Encode() []byte {
buf := bytes.NewBuffer(nil)
// set is primary
binary.Write(buf, binary.BigEndian, uint8(1))
l.themisLock.write(buf)
// write secondaries
binary.Write(buf, binary.BigEndian, int32(len(l.secondaries)))
for k, v := range l.secondaries {
c := &hbase.ColumnCoordinate{}
// TODO: handle error, now just log
if err := c.ParseFromString(k); err != nil {
log.Warnf("parse from string error, column coordinate: %s, secondary: %s, error: %v", c, k, err)
}
// TODO: handle error, now just log
if err := c.Write(buf); err != nil {
log.Warnf("write error, column coordinate: %s, buf: %s, error: %v", c, buf, err)
}
buf.WriteByte(uint8(v))
}
return buf.Bytes()
}
func (l *themisPrimaryLock) IsExpired() bool {
return l.themisLock.expired
}
func (l *themisPrimaryLock) getSecondaryColumnType(c *hbase.ColumnCoordinate) hbase.Type {
v, ok := l.secondaries[c.String()]
if !ok {
return hbase.TypeMinimum
}
return v
}
func (l *themisPrimaryLock) Role() LockRole {
return RolePrimary
}
func (l *themisPrimaryLock) addSecondary(col *hbase.ColumnCoordinate, t hbase.Type) {
l.secondaries[col.String()] = t
}
func (l *themisPrimaryLock) parse(buf iohelper.ByteMultiReader) error {
l.themisLock.parse(buf)
var sz int32
err := binary.Read(buf, binary.BigEndian, &sz)
if err != nil {
return errors.Trace(err)
}
for i := 0; i < int(sz); i++ {
c := &hbase.ColumnCoordinate{}
c.ParseField(buf)
b, err := buf.ReadByte()
if err != nil {
return errors.Trace(err)
}
t := hbase.Type(b)
l.addSecondary(c, t)
}
return nil
}

368
vendor/github.com/pingcap/go-themis/themis_rpc.go generated vendored Normal file
View file

@ -0,0 +1,368 @@
package themis
import (
"fmt"
"runtime/debug"
pb "github.com/golang/protobuf/proto"
"github.com/juju/errors"
"github.com/ngaut/log"
"github.com/pingcap/go-hbase"
"github.com/pingcap/go-hbase/proto"
"github.com/pingcap/go-themis/oracle"
)
func newThemisRPC(client hbase.HBaseClient, oracle oracle.Oracle, conf TxnConfig) *themisRPC {
return &themisRPC{
client: client,
conf: conf,
oracle: oracle,
}
}
type themisRPC struct {
client hbase.HBaseClient
conf TxnConfig
oracle oracle.Oracle
}
func (rpc *themisRPC) call(methodName string, tbl, row []byte, req pb.Message, resp pb.Message) error {
param, _ := pb.Marshal(req)
call := &hbase.CoprocessorServiceCall{
Row: row,
ServiceName: ThemisServiceName,
MethodName: methodName,
RequestParam: param,
}
r, err := rpc.client.ServiceCall(string(tbl), call)
if err != nil {
return errors.Trace(err)
}
err = pb.Unmarshal(r.GetValue().GetValue(), resp)
if err != nil {
return errors.Trace(err)
}
return nil
}
func (rpc *themisRPC) checkAndSetLockIsExpired(lock Lock) (bool, error) {
expired := rpc.oracle.IsExpired(lock.Timestamp(), rpc.conf.TTLInMs)
lock.SetExpired(expired)
return expired, nil
}
func (rpc *themisRPC) themisGet(tbl []byte, g *hbase.Get, startTs uint64, ignoreLock bool) (*hbase.ResultRow, error) {
req := &ThemisGetRequest{
Get: g.ToProto().(*proto.Get),
StartTs: pb.Uint64(startTs),
IgnoreLock: pb.Bool(ignoreLock),
}
var resp proto.Result
err := rpc.call("themisGet", tbl, g.Row, req, &resp)
if err != nil {
return nil, errors.Trace(err)
}
return hbase.NewResultRow(&resp), nil
}
func (rpc *themisRPC) themisBatchGet(tbl []byte, gets []*hbase.Get, startTs uint64, ignoreLock bool) ([]*hbase.ResultRow, error) {
var protoGets []*proto.Get
for _, g := range gets {
protoGets = append(protoGets, g.ToProto().(*proto.Get))
}
req := &ThemisBatchGetRequest{
Gets: protoGets,
StartTs: pb.Uint64(startTs),
IgnoreLock: pb.Bool(ignoreLock),
}
var resp ThemisBatchGetResponse
err := rpc.call("themisBatchGet", tbl, gets[0].Row, req, &resp)
if err != nil {
return nil, errors.Trace(err)
}
var results []*hbase.ResultRow
for _, rs := range resp.GetRs() {
results = append(results, hbase.NewResultRow(rs))
}
return results, nil
}
func (rpc *themisRPC) prewriteRow(tbl []byte, row []byte, mutations []*columnMutation, prewriteTs uint64, primaryLockBytes []byte, secondaryLockBytes []byte, primaryOffset int) (Lock, error) {
var cells []*proto.Cell
request := &ThemisPrewriteRequest{
PrewriteTs: pb.Uint64(prewriteTs),
PrimaryLock: primaryLockBytes,
SecondaryLock: secondaryLockBytes,
PrimaryIndex: pb.Int(primaryOffset),
}
request.ThemisPrewrite = &ThemisPrewrite{
Row: row,
}
if primaryLockBytes == nil {
request.PrimaryLock = []byte("")
}
if secondaryLockBytes == nil {
request.SecondaryLock = []byte("")
}
for _, m := range mutations {
cells = append(cells, m.toCell())
}
request.ThemisPrewrite.Mutations = cells
var res ThemisPrewriteResponse
err := rpc.call("prewriteRow", tbl, row, request, &res)
if err != nil {
return nil, errors.Trace(err)
}
b := res.ThemisPrewriteResult
if b == nil {
// if lock is empty, means we got the lock, otherwise some one else had
// locked this row, and the lock should return in rpc result
return nil, nil
}
// Oops, someone else have already locked this row.
commitTs := b.GetNewerWriteTs()
if commitTs != 0 {
log.Errorf("write conflict, encounter write with larger timestamp than prewriteTs=%d, commitTs=%d, row=%s", prewriteTs, commitTs, string(row))
return nil, ErrRetryable
}
l, err := parseLockFromBytes(b.ExistLock)
if err != nil {
return nil, errors.Trace(err)
}
col := &hbase.ColumnCoordinate{
Table: tbl,
Row: row,
Column: hbase.Column{
Family: b.Family,
Qual: b.Qualifier,
},
}
l.SetCoordinate(col)
return l, nil
}
func (rpc *themisRPC) isLockExpired(tbl, row []byte, ts uint64) (bool, error) {
req := &LockExpiredRequest{
Timestamp: pb.Uint64(ts),
}
var res LockExpiredResponse
if row == nil {
debug.PrintStack()
}
err := rpc.call("isLockExpired", tbl, row, req, &res)
if err != nil {
return false, errors.Trace(err)
}
return res.GetExpired(), nil
}
func (rpc *themisRPC) getLockAndErase(cc *hbase.ColumnCoordinate, prewriteTs uint64) (Lock, error) {
req := &EraseLockRequest{
Row: cc.Row,
Family: cc.Column.Family,
Qualifier: cc.Column.Qual,
PrewriteTs: pb.Uint64(prewriteTs),
}
var res EraseLockResponse
err := rpc.call("getLockAndErase", cc.Table, cc.Row, req, &res)
if err != nil {
return nil, errors.Trace(err)
}
b := res.GetLock()
if len(b) == 0 {
return nil, nil
}
return parseLockFromBytes(b)
}
func (rpc *themisRPC) commitRow(tbl, row []byte, mutations []*columnMutation,
prewriteTs, commitTs uint64, primaryOffset int) error {
req := &ThemisCommitRequest{}
req.ThemisCommit = &ThemisCommit{
Row: row,
PrewriteTs: pb.Uint64(prewriteTs),
CommitTs: pb.Uint64(commitTs),
PrimaryIndex: pb.Int(primaryOffset),
}
for _, m := range mutations {
req.ThemisCommit.Mutations = append(req.ThemisCommit.Mutations, m.toCell())
}
var res ThemisCommitResponse
err := rpc.call("commitRow", tbl, row, req, &res)
if err != nil {
return errors.Trace(err)
}
ok := res.GetResult()
if !ok {
if primaryOffset == -1 {
return errors.Errorf("commit secondary failed, tbl: %s row: %q ts: %d", tbl, row, commitTs)
}
return errors.Errorf("commit primary failed, tbl: %s row: %q ts: %d", tbl, row, commitTs)
}
return nil
}
func (rpc *themisRPC) batchCommitSecondaryRows(tbl []byte, rowMs map[string]*rowMutation, prewriteTs, commitTs uint64) error {
req := &ThemisBatchCommitSecondaryRequest{}
i := 0
var lastRow []byte
req.ThemisCommit = make([]*ThemisCommit, len(rowMs))
for row, rowM := range rowMs {
var cells []*proto.Cell
for col, m := range rowM.mutations {
cells = append(cells, toCellFromRowM(col, m))
}
req.ThemisCommit[i] = &ThemisCommit{
Row: []byte(row),
Mutations: cells,
PrewriteTs: pb.Uint64(prewriteTs),
CommitTs: pb.Uint64(commitTs),
PrimaryIndex: pb.Int(-1),
}
i++
lastRow = []byte(row)
}
var res ThemisBatchCommitSecondaryResponse
err := rpc.call("batchCommitSecondaryRows", tbl, lastRow, req, &res)
if err != nil {
return errors.Trace(err)
}
log.Info("call batch commit secondary rows", len(req.ThemisCommit))
cResult := res.BatchCommitSecondaryResult
if cResult != nil && len(cResult) > 0 {
errorInfo := "commit failed, tbl:" + string(tbl)
for _, r := range cResult {
errorInfo += (" row:" + string(r.Row))
}
return errors.New(fmt.Sprintf("%s, commitTs:%d", errorInfo, commitTs))
}
return nil
}
func (rpc *themisRPC) commitSecondaryRow(tbl, row []byte, mutations []*columnMutation,
prewriteTs, commitTs uint64) error {
return rpc.commitRow(tbl, row, mutations, prewriteTs, commitTs, -1)
}
func (rpc *themisRPC) prewriteSecondaryRow(tbl, row []byte,
mutations []*columnMutation, prewriteTs uint64,
secondaryLockBytes []byte) (Lock, error) {
return rpc.prewriteRow(tbl, row, mutations, prewriteTs, nil, secondaryLockBytes, -1)
}
func (rpc *themisRPC) batchPrewriteSecondaryRows(tbl []byte, rowMs map[string]*rowMutation, prewriteTs uint64, secondaryLockBytes []byte) (map[string]Lock, error) {
request := &ThemisBatchPrewriteSecondaryRequest{
PrewriteTs: pb.Uint64(prewriteTs),
SecondaryLock: secondaryLockBytes,
}
request.ThemisPrewrite = make([]*ThemisPrewrite, len(rowMs))
if secondaryLockBytes == nil {
secondaryLockBytes = []byte("")
}
i := 0
var lastRow []byte
for row, rowM := range rowMs {
var cells []*proto.Cell
for col, m := range rowM.mutations {
cells = append(cells, toCellFromRowM(col, m))
}
request.ThemisPrewrite[i] = &ThemisPrewrite{
Row: []byte(row),
Mutations: cells,
}
i++
lastRow = []byte(row)
}
var res ThemisBatchPrewriteSecondaryResponse
err := rpc.call("batchPrewriteSecondaryRows", tbl, lastRow, request, &res)
if err != nil {
return nil, errors.Trace(err)
}
//Perhaps, part row has not in a region, sample : when region split, then need try
lockMap := make(map[string]Lock)
if res.RowsNotInRegion != nil && len(res.RowsNotInRegion) > 0 {
for _, r := range res.RowsNotInRegion {
tl, err := rpc.prewriteSecondaryRow(tbl, r, rowMs[string(r)].mutationList(true), prewriteTs, secondaryLockBytes)
if err != nil {
return nil, errors.Trace(err)
}
if tl != nil {
lockMap[string(r)] = tl
}
}
}
b := res.ThemisPrewriteResult
if b != nil && len(b) > 0 {
for _, pResult := range b {
lock, err := judgePerwriteResultRow(pResult, tbl, prewriteTs, pResult.Row)
if err != nil {
return nil, errors.Trace(err)
}
if lock != nil {
lockMap[string(pResult.Row)] = lock
}
}
}
return lockMap, nil
}
func judgePerwriteResultRow(pResult *ThemisPrewriteResult, tbl []byte, prewriteTs uint64, row []byte) (Lock, error) {
// Oops, someone else have already locked this row.
newerTs := pResult.GetNewerWriteTs()
if newerTs != 0 {
return nil, ErrRetryable
}
l, err := parseLockFromBytes(pResult.ExistLock)
if err != nil {
return nil, errors.Trace(err)
}
col := &hbase.ColumnCoordinate{
Table: tbl,
Row: row,
Column: hbase.Column{
Family: pResult.Family,
Qual: pResult.Qualifier,
},
}
l.SetCoordinate(col)
return l, nil
}
func toCellFromRowM(col string, cvPair *mutationValuePair) *proto.Cell {
c := &hbase.Column{}
// TODO: handle error, now just log
if err := c.ParseFromString(col); err != nil {
log.Warnf("parse from string error, column: %s, col: %s, error: %v", c, col, err)
}
ret := &proto.Cell{
Family: c.Family,
Qualifier: c.Qual,
Value: cvPair.value,
}
if cvPair.typ == hbase.TypePut { // put
ret.CellType = proto.CellType_PUT.Enum()
} else if cvPair.typ == hbase.TypeMinimum { // onlyLock
ret.CellType = proto.CellType_MINIMUM.Enum()
} else { // delete, themis delete API only support delete column
ret.CellType = proto.CellType_DELETE_COLUMN.Enum()
}
return ret
}

85
vendor/github.com/pingcap/go-themis/themis_scan.go generated vendored Normal file
View file

@ -0,0 +1,85 @@
package themis
import (
"bytes"
"encoding/binary"
"github.com/ngaut/log"
"github.com/pingcap/go-hbase"
)
type ThemisScanner struct {
scan *hbase.Scan
txn *themisTxn
tbl []byte
}
func newThemisScanner(tbl []byte, txn *themisTxn, batchSize int, c hbase.HBaseClient) *ThemisScanner {
s := hbase.NewScan(tbl, batchSize, c)
// add start ts
b := bytes.NewBuffer(nil)
binary.Write(b, binary.BigEndian, txn.startTs)
s.AddAttr("_themisTransationStartTs_", b.Bytes())
return &ThemisScanner{
scan: s,
txn: txn,
tbl: tbl,
}
}
func (s *ThemisScanner) setStartRow(start []byte) {
s.scan.StartRow = start
}
func (s *ThemisScanner) setStopRow(stop []byte) {
s.scan.StopRow = stop
}
func (s *ThemisScanner) SetTimeRange(tsRangeFrom uint64, tsRangeTo uint64) {
s.scan.TsRangeFrom = tsRangeFrom
s.scan.TsRangeTo = tsRangeTo
}
func (s *ThemisScanner) SetMaxVersions(maxVersions uint32) {
s.scan.MaxVersions = maxVersions
}
func (s *ThemisScanner) createGetFromScan(row []byte) *hbase.Get {
return s.scan.CreateGetFromScan(row)
}
func (s *ThemisScanner) Next() *hbase.ResultRow {
r := s.scan.Next()
if r == nil {
return nil
}
// if we encounter conflict locks, we need to clean lock for this row and read again
if isLockResult(r) {
g := s.createGetFromScan(r.Row)
r, err := s.txn.tryToCleanLockAndGetAgain(s.tbl, g, r.SortedColumns)
if err != nil {
log.Error(err)
return nil
}
// empty result indicates the current row has been erased, we should get next row
if r == nil {
return s.Next()
} else {
return r
}
}
return r
}
func (s *ThemisScanner) Closed() bool {
return s.scan.Closed()
}
func (s *ThemisScanner) Close() {
if !s.scan.Closed() {
// TODO: handle error, now just log
if err := s.scan.Close(); err != nil {
log.Warnf("scanner close error, scan: %s, error: %v", s.scan, err)
}
}
}

View file

@ -0,0 +1,64 @@
package themis
import (
"bytes"
"encoding/binary"
"github.com/juju/errors"
"github.com/ngaut/log"
"github.com/pingcap/go-hbase"
"github.com/pingcap/go-hbase/iohelper"
)
type themisSecondaryLock struct {
*themisLock
primaryCoordinate *hbase.ColumnCoordinate
}
func newThemisSecondaryLock() *themisSecondaryLock {
return &themisSecondaryLock{
themisLock: &themisLock{
clientAddr: "null",
},
primaryCoordinate: &hbase.ColumnCoordinate{},
}
}
func (l *themisSecondaryLock) Primary() Lock {
pl := newThemisPrimaryLock()
pl.coordinate = l.primaryCoordinate
pl.ts = l.ts
pl.clientAddr = l.clientAddr
pl.addSecondary(l.coordinate, l.typ)
return pl
}
func (l *themisSecondaryLock) Secondaries() []Lock {
return nil
}
func (l *themisSecondaryLock) Role() LockRole {
return RoleSecondary
}
func (l *themisSecondaryLock) Encode() []byte {
buf := bytes.NewBuffer(nil)
binary.Write(buf, binary.BigEndian, uint8(0))
l.themisLock.write(buf)
// TODO: handle error, now just log
if err := l.primaryCoordinate.Write(buf); err != nil {
log.Warnf("write error, primary coordinate: %s, buf: %s, err: %v", l, buf, err)
}
return buf.Bytes()
}
func (l *themisSecondaryLock) parse(r iohelper.ByteMultiReader) error {
l.themisLock.parse(r)
primary := &hbase.ColumnCoordinate{}
err := primary.ParseField(r)
if err != nil {
return errors.Trace(err)
}
l.primaryCoordinate = primary
return nil
}

796
vendor/github.com/pingcap/go-themis/themis_txn.go generated vendored Normal file
View file

@ -0,0 +1,796 @@
package themis
import (
"fmt"
"sync"
"time"
"github.com/juju/errors"
"github.com/ngaut/log"
"github.com/pingcap/go-hbase"
"github.com/pingcap/go-themis/oracle"
)
type TxnConfig struct {
ConcurrentPrewriteAndCommit bool
WaitSecondaryCommit bool
TTLInMs uint64
MaxRowsInOneTxn int
// options below is for debugging and testing
brokenPrewriteSecondaryTest bool
brokenPrewriteSecondaryAndRollbackTest bool
brokenCommitPrimaryTest bool
brokenCommitSecondaryTest bool
}
var defaultTxnConf = TxnConfig{
ConcurrentPrewriteAndCommit: true,
WaitSecondaryCommit: false,
MaxRowsInOneTxn: 50000,
TTLInMs: 5 * 1000, // default txn TTL: 5s
brokenPrewriteSecondaryTest: false,
brokenPrewriteSecondaryAndRollbackTest: false,
brokenCommitPrimaryTest: false,
brokenCommitSecondaryTest: false,
}
type themisTxn struct {
client hbase.HBaseClient
rpc *themisRPC
lockCleaner LockManager
oracle oracle.Oracle
mutationCache *columnMutationCache
startTs uint64
commitTs uint64
primaryRow *rowMutation
primary *hbase.ColumnCoordinate
secondaryRows []*rowMutation
secondary []*hbase.ColumnCoordinate
primaryRowOffset int
singleRowTxn bool
secondaryLockBytes []byte
conf TxnConfig
hooks *txnHook
}
var _ Txn = (*themisTxn)(nil)
var (
// ErrSimulated is used when maybe rollback occurs error too.
ErrSimulated = errors.New("simulated error")
maxCleanLockRetryCount = 30
pauseTime = 300 * time.Millisecond
)
func NewTxn(c hbase.HBaseClient, oracle oracle.Oracle) (Txn, error) {
return NewTxnWithConf(c, defaultTxnConf, oracle)
}
func NewTxnWithConf(c hbase.HBaseClient, conf TxnConfig, oracle oracle.Oracle) (Txn, error) {
var err error
txn := &themisTxn{
client: c,
mutationCache: newColumnMutationCache(),
oracle: oracle,
primaryRowOffset: -1,
conf: conf,
rpc: newThemisRPC(c, oracle, conf),
hooks: newHook(),
}
txn.startTs, err = txn.oracle.GetTimestamp()
if err != nil {
return nil, errors.Trace(err)
}
txn.lockCleaner = newThemisLockManager(txn.rpc, c)
return txn, nil
}
func (txn *themisTxn) setHook(hooks *txnHook) {
txn.hooks = hooks
}
func (txn *themisTxn) Gets(tbl string, gets []*hbase.Get) ([]*hbase.ResultRow, error) {
results, err := txn.rpc.themisBatchGet([]byte(tbl), gets, txn.startTs, false)
if err != nil {
return nil, errors.Trace(err)
}
var ret []*hbase.ResultRow
hasLock := false
for _, r := range results {
// if this row is locked, try clean lock and get again
if isLockResult(r) {
hasLock = true
err = txn.constructLockAndClean([]byte(tbl), r.SortedColumns)
if err != nil {
// TODO if it's a conflict error, it means this lock
// isn't expired, maybe we can retry or return partial results.
return nil, errors.Trace(err)
}
}
// it's OK, because themisBatchGet doesn't return nil value.
ret = append(ret, r)
}
if hasLock {
// after we cleaned locks, try to get again.
ret, err = txn.rpc.themisBatchGet([]byte(tbl), gets, txn.startTs, true)
if err != nil {
return nil, errors.Trace(err)
}
}
return ret, nil
}
func (txn *themisTxn) Get(tbl string, g *hbase.Get) (*hbase.ResultRow, error) {
r, err := txn.rpc.themisGet([]byte(tbl), g, txn.startTs, false)
if err != nil {
return nil, errors.Trace(err)
}
// contain locks, try to clean and get again
if r != nil && isLockResult(r) {
r, err = txn.tryToCleanLockAndGetAgain([]byte(tbl), g, r.SortedColumns)
if err != nil {
return nil, errors.Trace(err)
}
}
return r, nil
}
func (txn *themisTxn) Put(tbl string, p *hbase.Put) {
// add mutation to buffer
for _, e := range getEntriesFromPut(p) {
txn.mutationCache.addMutation([]byte(tbl), p.Row, e.Column, e.typ, e.value, false)
}
}
func (txn *themisTxn) Delete(tbl string, p *hbase.Delete) error {
entries, err := getEntriesFromDel(p)
if err != nil {
return errors.Trace(err)
}
for _, e := range entries {
txn.mutationCache.addMutation([]byte(tbl), p.Row, e.Column, e.typ, e.value, false)
}
return nil
}
func (txn *themisTxn) Commit() error {
if txn.mutationCache.getMutationCount() == 0 {
return nil
}
if txn.mutationCache.getRowCount() > txn.conf.MaxRowsInOneTxn {
return ErrTooManyRows
}
txn.selectPrimaryAndSecondaries()
err := txn.prewritePrimary()
if err != nil {
// no need to check wrong region here, hbase client will retry when
// occurs single row NotInRegion error.
log.Error(errors.ErrorStack(err))
// it's safe to retry, because this transaction is not committed.
return ErrRetryable
}
err = txn.prewriteSecondary()
if err != nil {
if isWrongRegionErr(err) {
log.Warn("region info outdated")
// reset hbase client buffered region info
txn.client.CleanAllRegionCache()
}
return ErrRetryable
}
txn.commitTs, err = txn.oracle.GetTimestamp()
if err != nil {
log.Error(errors.ErrorStack(err))
return ErrRetryable
}
err = txn.commitPrimary()
if err != nil {
// commit primary error, rollback
log.Error("commit primary row failed", txn.startTs, err)
txn.rollbackRow(txn.primaryRow.tbl, txn.primaryRow)
txn.rollbackSecondaryRow(len(txn.secondaryRows) - 1)
return ErrRetryable
}
txn.commitSecondary()
log.Debug("themis txn commit successfully", txn.startTs, txn.commitTs)
return nil
}
func (txn *themisTxn) commitSecondary() {
if bypass, _, _ := txn.hooks.beforeCommitSecondary(txn, nil); !bypass {
return
}
if txn.conf.brokenCommitSecondaryTest {
txn.brokenCommitSecondary()
return
}
if txn.conf.ConcurrentPrewriteAndCommit {
txn.batchCommitSecondary(txn.conf.WaitSecondaryCommit)
} else {
txn.commitSecondarySync()
}
}
func (txn *themisTxn) commitSecondarySync() {
for _, r := range txn.secondaryRows {
err := txn.rpc.commitSecondaryRow(r.tbl, r.row, r.mutationList(false), txn.startTs, txn.commitTs)
if err != nil {
// fail of secondary commit will not stop the commits of next
// secondaries
log.Warning(err)
}
}
}
func (txn *themisTxn) batchCommitSecondary(wait bool) error {
//will batch commit all rows in a region
rsRowMap, err := txn.groupByRegion()
if err != nil {
return errors.Trace(err)
}
wg := sync.WaitGroup{}
for _, regionRowMap := range rsRowMap {
wg.Add(1)
_, firstRowM := getFirstEntity(regionRowMap)
go func(cli *themisRPC, tbl string, rMap map[string]*rowMutation, startTs, commitTs uint64) {
defer wg.Done()
err := cli.batchCommitSecondaryRows([]byte(tbl), rMap, startTs, commitTs)
if err != nil {
// fail of secondary commit will not stop the commits of next
// secondaries
if isWrongRegionErr(err) {
txn.client.CleanAllRegionCache()
log.Warn("region info outdated when committing secondary rows, don't panic")
}
}
}(txn.rpc, string(firstRowM.tbl), regionRowMap, txn.startTs, txn.commitTs)
}
if wait {
wg.Wait()
}
return nil
}
func (txn *themisTxn) groupByRegion() (map[string]map[string]*rowMutation, error) {
rsRowMap := make(map[string]map[string]*rowMutation)
for _, rm := range txn.secondaryRows {
region, err := txn.client.LocateRegion(rm.tbl, rm.row, true)
if err != nil {
return nil, errors.Trace(err)
}
key := getBatchGroupKey(region, string(rm.tbl))
if _, exists := rsRowMap[key]; !exists {
rsRowMap[key] = map[string]*rowMutation{}
}
rsRowMap[key][string(rm.row)] = rm
}
return rsRowMap, nil
}
func (txn *themisTxn) commitPrimary() error {
if txn.conf.brokenCommitPrimaryTest {
return txn.brokenCommitPrimary()
}
return txn.rpc.commitRow(txn.primary.Table, txn.primary.Row,
txn.primaryRow.mutationList(false),
txn.startTs, txn.commitTs, txn.primaryRowOffset)
}
func (txn *themisTxn) selectPrimaryAndSecondaries() {
txn.secondary = nil
for tblName, rowMutations := range txn.mutationCache.mutations {
for _, rowMutation := range rowMutations {
row := rowMutation.row
findPrimaryInRow := false
for i, mutation := range rowMutation.mutationList(true) {
colcord := hbase.NewColumnCoordinate([]byte(tblName), row, mutation.Family, mutation.Qual)
// set the first column as primary if primary is not set by user
if txn.primaryRowOffset == -1 &&
(txn.primary == nil || txn.primary.Equal(colcord)) {
txn.primary = colcord
txn.primaryRowOffset = i
txn.primaryRow = rowMutation
findPrimaryInRow = true
} else {
txn.secondary = append(txn.secondary, colcord)
}
}
if !findPrimaryInRow {
txn.secondaryRows = append(txn.secondaryRows, rowMutation)
}
}
}
// hook for test
if bypass, _, _ := txn.hooks.afterChoosePrimaryAndSecondary(txn, nil); !bypass {
return
}
if len(txn.secondaryRows) == 0 {
txn.singleRowTxn = true
}
// construct secondary lock
secondaryLock := txn.constructSecondaryLock(hbase.TypePut)
if secondaryLock != nil {
txn.secondaryLockBytes = secondaryLock.Encode()
} else {
txn.secondaryLockBytes = nil
}
}
func (txn *themisTxn) constructSecondaryLock(typ hbase.Type) *themisSecondaryLock {
if txn.primaryRow.getSize() <= 1 && len(txn.secondaryRows) == 0 {
return nil
}
l := newThemisSecondaryLock()
l.primaryCoordinate = txn.primary
l.ts = txn.startTs
// TODO set client addr
return l
}
func (txn *themisTxn) constructPrimaryLock() *themisPrimaryLock {
l := newThemisPrimaryLock()
l.typ = txn.primaryRow.getType(txn.primary.Column)
l.ts = txn.startTs
for _, c := range txn.secondary {
l.addSecondary(c, txn.mutationCache.getMutation(c).typ)
}
return l
}
func (txn *themisTxn) constructLockAndClean(tbl []byte, lockKvs []*hbase.Kv) error {
locks, err := getLocksFromResults([]byte(tbl), lockKvs, txn.rpc)
if err != nil {
return errors.Trace(err)
}
for _, lock := range locks {
err := txn.cleanLockWithRetry(lock)
if err != nil {
return errors.Trace(err)
}
}
return nil
}
func (txn *themisTxn) tryToCleanLockAndGetAgain(tbl []byte, g *hbase.Get, lockKvs []*hbase.Kv) (*hbase.ResultRow, error) {
// try to clean locks
err := txn.constructLockAndClean(tbl, lockKvs)
if err != nil {
return nil, errors.Trace(err)
}
// get again, ignore lock
r, err := txn.rpc.themisGet([]byte(tbl), g, txn.startTs, true)
if err != nil {
return nil, errors.Trace(err)
}
return r, nil
}
func (txn *themisTxn) commitSecondaryAndCleanLock(lock *themisSecondaryLock, commitTs uint64) error {
cc := lock.Coordinate()
mutation := &columnMutation{
Column: &cc.Column,
mutationValuePair: &mutationValuePair{
typ: lock.typ,
},
}
err := txn.rpc.commitSecondaryRow(cc.Table, cc.Row,
[]*columnMutation{mutation}, lock.Timestamp(), commitTs)
if err != nil {
return errors.Trace(err)
}
return nil
}
func (txn *themisTxn) cleanLockWithRetry(lock Lock) error {
for i := 0; i < maxCleanLockRetryCount; i++ {
if exists, err := txn.lockCleaner.IsLockExists(lock.Coordinate(), 0, lock.Timestamp()); err != nil || !exists {
return errors.Trace(err)
}
log.Warnf("lock exists txn: %v lock-txn: %v row: %q", txn.startTs, lock.Timestamp(), lock.Coordinate().Row)
// try clean lock
err := txn.tryToCleanLock(lock)
if errorEqual(err, ErrLockNotExpired) {
log.Warn("sleep a while, and retry clean lock", txn.startTs)
// TODO(dongxu) use cleverer retry sleep time interval
time.Sleep(pauseTime)
continue
} else if err != nil {
return errors.Trace(err)
}
// lock cleaned successfully
return nil
}
return ErrCleanLockFailed
}
func (txn *themisTxn) tryToCleanLock(lock Lock) error {
// if it's secondary lock, first we'll check if its primary lock has been released.
if lock.Role() == RoleSecondary {
// get primary lock
pl := lock.Primary()
// check primary lock is exists
exists, err := txn.lockCleaner.IsLockExists(pl.Coordinate(), 0, pl.Timestamp())
if err != nil {
return errors.Trace(err)
}
if !exists {
// primary row is committed, commit this row
cc := pl.Coordinate()
commitTs, err := txn.lockCleaner.GetCommitTimestamp(cc, pl.Timestamp())
if err != nil {
return errors.Trace(err)
}
if commitTs > 0 {
// if this transction has been committed
log.Info("txn has been committed, ts:", commitTs, "prewriteTs:", pl.Timestamp())
// commit secondary rows
err := txn.commitSecondaryAndCleanLock(lock.(*themisSecondaryLock), commitTs)
if err != nil {
return errors.Trace(err)
}
return nil
}
}
}
expired, err := txn.rpc.checkAndSetLockIsExpired(lock)
if err != nil {
return errors.Trace(err)
}
// only clean expired lock
if expired {
// try to clean primary lock
pl := lock.Primary()
commitTs, cleanedLock, err := txn.lockCleaner.CleanLock(pl.Coordinate(), pl.Timestamp())
if err != nil {
return errors.Trace(err)
}
if cleanedLock != nil {
pl = cleanedLock
}
log.Info("try clean secondary locks", pl.Timestamp())
// clean secondary locks
// erase lock and data if commitTs is 0; otherwise, commit it.
for k, v := range pl.(*themisPrimaryLock).secondaries {
cc := &hbase.ColumnCoordinate{}
if err = cc.ParseFromString(k); err != nil {
return errors.Trace(err)
}
if commitTs == 0 {
// commitTs == 0, means clean primary lock successfully
// expire trx havn't committed yet, we must delete lock and
// dirty data
err = txn.lockCleaner.EraseLockAndData(cc, pl.Timestamp())
if err != nil {
return errors.Trace(err)
}
} else {
// primary row is committed, so we must commit other
// secondary rows
mutation := &columnMutation{
Column: &cc.Column,
mutationValuePair: &mutationValuePair{
typ: v,
},
}
err = txn.rpc.commitSecondaryRow(cc.Table, cc.Row,
[]*columnMutation{mutation}, pl.Timestamp(), commitTs)
if err != nil {
return errors.Trace(err)
}
}
}
} else {
return ErrLockNotExpired
}
return nil
}
func (txn *themisTxn) batchPrewriteSecondaryRowsWithLockClean(tbl []byte, rowMs map[string]*rowMutation) error {
locks, err := txn.batchPrewriteSecondaryRows(tbl, rowMs)
if err != nil {
return errors.Trace(err)
}
// lock clean
if locks != nil && len(locks) > 0 {
// hook for test
if bypass, _, err := txn.hooks.onSecondaryOccursLock(txn, locks); !bypass {
return errors.Trace(err)
}
// try one more time after clean lock successfully
for _, lock := range locks {
err = txn.cleanLockWithRetry(lock)
if err != nil {
return errors.Trace(err)
}
// prewrite all secondary rows
locks, err = txn.batchPrewriteSecondaryRows(tbl, rowMs)
if err != nil {
return errors.Trace(err)
}
if len(locks) > 0 {
for _, l := range locks {
log.Errorf("can't clean lock, column:%q; conflict lock: %+v, lock ts: %d", l.Coordinate(), l, l.Timestamp())
}
return ErrRetryable
}
}
}
return nil
}
func (txn *themisTxn) prewriteRowWithLockClean(tbl []byte, mutation *rowMutation, containPrimary bool) error {
lock, err := txn.prewriteRow(tbl, mutation, containPrimary)
if err != nil {
return errors.Trace(err)
}
// lock clean
if lock != nil {
// hook for test
if bypass, _, err := txn.hooks.beforePrewriteLockClean(txn, lock); !bypass {
return errors.Trace(err)
}
err = txn.cleanLockWithRetry(lock)
if err != nil {
return errors.Trace(err)
}
// try one more time after clean lock successfully
lock, err = txn.prewriteRow(tbl, mutation, containPrimary)
if err != nil {
return errors.Trace(err)
}
if lock != nil {
log.Errorf("can't clean lock, column:%q; conflict lock: %+v, lock ts: %d", lock.Coordinate(), lock, lock.Timestamp())
return ErrRetryable
}
}
return nil
}
func (txn *themisTxn) batchPrewriteSecondaryRows(tbl []byte, rowMs map[string]*rowMutation) (map[string]Lock, error) {
return txn.rpc.batchPrewriteSecondaryRows(tbl, rowMs, txn.startTs, txn.secondaryLockBytes)
}
func (txn *themisTxn) prewriteRow(tbl []byte, mutation *rowMutation, containPrimary bool) (Lock, error) {
// hook for test
if bypass, ret, err := txn.hooks.onPrewriteRow(txn, []interface{}{mutation, containPrimary}); !bypass {
return ret.(Lock), errors.Trace(err)
}
if containPrimary {
// try to get lock
return txn.rpc.prewriteRow(tbl, mutation.row,
mutation.mutationList(true),
txn.startTs,
txn.constructPrimaryLock().Encode(),
txn.secondaryLockBytes, txn.primaryRowOffset)
}
return txn.rpc.prewriteSecondaryRow(tbl, mutation.row,
mutation.mutationList(true),
txn.startTs,
txn.secondaryLockBytes)
}
func (txn *themisTxn) prewritePrimary() error {
// hook for test
if bypass, _, err := txn.hooks.beforePrewritePrimary(txn, nil); !bypass {
return err
}
err := txn.prewriteRowWithLockClean(txn.primary.Table, txn.primaryRow, true)
if err != nil {
log.Debugf("prewrite primary %v %q failed: %v", txn.startTs, txn.primaryRow.row, err.Error())
return errors.Trace(err)
}
log.Debugf("prewrite primary %v %q successfully", txn.startTs, txn.primaryRow.row)
return nil
}
func (txn *themisTxn) prewriteSecondary() error {
// hook for test
if bypass, _, err := txn.hooks.beforePrewriteSecondary(txn, nil); !bypass {
return err
}
if txn.conf.brokenPrewriteSecondaryTest {
return txn.brokenPrewriteSecondary()
}
if txn.conf.ConcurrentPrewriteAndCommit {
return txn.batchPrewriteSecondaries()
}
return txn.prewriteSecondarySync()
}
func (txn *themisTxn) prewriteSecondarySync() error {
for i, mu := range txn.secondaryRows {
err := txn.prewriteRowWithLockClean(mu.tbl, mu, false)
if err != nil {
// rollback
txn.rollbackRow(txn.primaryRow.tbl, txn.primaryRow)
txn.rollbackSecondaryRow(i)
return errors.Trace(err)
}
}
return nil
}
// just for test
func (txn *themisTxn) brokenCommitPrimary() error {
// do nothing
log.Warn("Simulating primary commit failed")
return nil
}
// just for test
func (txn *themisTxn) brokenCommitSecondary() {
// do nothing
log.Warn("Simulating secondary commit failed")
}
func (txn *themisTxn) brokenPrewriteSecondary() error {
log.Warn("Simulating prewrite secondary failed")
for i, rm := range txn.secondaryRows {
if i == len(txn.secondary)-1 {
if !txn.conf.brokenPrewriteSecondaryAndRollbackTest {
// simulating prewrite failed, need rollback
txn.rollbackRow(txn.primaryRow.tbl, txn.primaryRow)
txn.rollbackSecondaryRow(i)
}
// maybe rollback occurs error too
return ErrSimulated
}
txn.prewriteRowWithLockClean(rm.tbl, rm, false)
}
return nil
}
func (txn *themisTxn) batchPrewriteSecondaries() error {
wg := sync.WaitGroup{}
//will batch prewrite all rows in a region
rsRowMap, err := txn.groupByRegion()
if err != nil {
return errors.Trace(err)
}
errChan := make(chan error, len(rsRowMap))
defer close(errChan)
successChan := make(chan map[string]*rowMutation, len(rsRowMap))
defer close(successChan)
for _, regionRowMap := range rsRowMap {
wg.Add(1)
_, firstRowM := getFirstEntity(regionRowMap)
go func(tbl []byte, rMap map[string]*rowMutation) {
defer wg.Done()
err := txn.batchPrewriteSecondaryRowsWithLockClean(tbl, rMap)
if err != nil {
errChan <- err
} else {
successChan <- rMap
}
}(firstRowM.tbl, regionRowMap)
}
wg.Wait()
if len(errChan) != 0 {
// occur error, clean success prewrite mutations
log.Warnf("batch prewrite secondary rows error, rolling back %d %d", len(successChan), txn.startTs)
txn.rollbackRow(txn.primaryRow.tbl, txn.primaryRow)
L:
for {
select {
case succMutMap := <-successChan:
{
for _, rowMut := range succMutMap {
txn.rollbackRow(rowMut.tbl, rowMut)
}
}
default:
break L
}
}
err := <-errChan
if err != nil {
log.Error("batch prewrite secondary rows error, txn:", txn.startTs, err)
}
return errors.Trace(err)
}
return nil
}
func getFirstEntity(rowMap map[string]*rowMutation) (string, *rowMutation) {
for row, rowM := range rowMap {
return row, rowM
}
return "", nil
}
func getBatchGroupKey(rInfo *hbase.RegionInfo, tblName string) string {
return rInfo.Server + "_" + rInfo.Name
}
func (txn *themisTxn) rollbackRow(tbl []byte, mutation *rowMutation) error {
l := fmt.Sprintf("\nrolling back %q %d {\n", mutation.row, txn.startTs)
for _, v := range mutation.getColumns() {
l += fmt.Sprintf("\t%s:%s\n", string(v.Family), string(v.Qual))
}
l += "}\n"
log.Warn(l)
for _, col := range mutation.getColumns() {
cc := &hbase.ColumnCoordinate{
Table: tbl,
Row: mutation.row,
Column: col,
}
err := txn.lockCleaner.EraseLockAndData(cc, txn.startTs)
if err != nil {
return errors.Trace(err)
}
}
return nil
}
func (txn *themisTxn) rollbackSecondaryRow(successIndex int) error {
for i := successIndex; i >= 0; i-- {
r := txn.secondaryRows[i]
err := txn.rollbackRow(r.tbl, r)
if err != nil {
return errors.Trace(err)
}
}
return nil
}
func (txn *themisTxn) GetScanner(tbl []byte, startKey, endKey []byte, batchSize int) *ThemisScanner {
scanner := newThemisScanner(tbl, txn, batchSize, txn.client)
if startKey != nil {
scanner.setStartRow(startKey)
}
if endKey != nil {
scanner.setStopRow(endKey)
}
return scanner
}
func (txn *themisTxn) Release() {
txn.primary = nil
txn.primaryRow = nil
txn.secondary = nil
txn.secondaryRows = nil
txn.startTs = 0
txn.commitTs = 0
}
func (txn *themisTxn) String() string {
return fmt.Sprintf("%d", txn.startTs)
}
func (txn *themisTxn) GetCommitTS() uint64 {
return txn.commitTs
}
func (txn *themisTxn) GetStartTS() uint64 {
return txn.startTs
}
func (txn *themisTxn) LockRow(tbl string, rowkey []byte) error {
g := hbase.NewGet(rowkey)
r, err := txn.Get(tbl, g)
if err != nil {
log.Warnf("get row error, table:%s, row:%q, error:%v", tbl, rowkey, err)
return errors.Trace(err)
}
if r == nil {
log.Warnf("has not data to lock, table:%s, row:%q", tbl, rowkey)
return nil
}
for _, v := range r.Columns {
txn.mutationCache.addMutation([]byte(tbl), rowkey, &v.Column, hbase.TypeMinimum, nil, true)
}
return nil
}

28
vendor/github.com/pingcap/go-themis/txn.go generated vendored Normal file
View file

@ -0,0 +1,28 @@
package themis
import (
"errors"
"github.com/pingcap/go-hbase"
)
var (
ErrLockNotExpired = errors.New("lock not expired")
ErrCleanLockFailed = errors.New("clean lock failed")
ErrWrongRegion = errors.New("wrong region, please retry")
ErrTooManyRows = errors.New("too many rows in one transaction")
ErrRetryable = errors.New("try again later")
)
type Txn interface {
Get(t string, get *hbase.Get) (*hbase.ResultRow, error)
Gets(t string, gets []*hbase.Get) ([]*hbase.ResultRow, error)
LockRow(t string, row []byte) error
Put(t string, put *hbase.Put)
Delete(t string, del *hbase.Delete) error
GetScanner(tbl []byte, startKey, endKey []byte, batchSize int) *ThemisScanner
Release()
Commit() error
GetStartTS() uint64
GetCommitTS() uint64
}

36
vendor/github.com/pingcap/go-themis/txn_hook.go generated vendored Normal file
View file

@ -0,0 +1,36 @@
package themis
// Hooks for debugging and testing
type fnHook func(txn *themisTxn, ctx interface{}) (bypass bool, ret interface{}, err error)
var emptyHookFn = func(txn *themisTxn, ctx interface{}) (bypass bool, ret interface{}, err error) {
return true, nil, nil
}
type txnHook struct {
afterChoosePrimaryAndSecondary fnHook
beforePrewritePrimary fnHook
beforePrewriteLockClean fnHook
beforePrewriteSecondary fnHook
beforeCommitPrimary fnHook
beforeCommitSecondary fnHook
onSecondaryOccursLock fnHook
onPrewriteRow fnHook
onTxnSuccess fnHook
onTxnFailed fnHook
}
func newHook() *txnHook {
return &txnHook{
afterChoosePrimaryAndSecondary: emptyHookFn,
beforePrewritePrimary: emptyHookFn,
beforePrewriteLockClean: emptyHookFn,
beforePrewriteSecondary: emptyHookFn,
beforeCommitPrimary: emptyHookFn,
beforeCommitSecondary: emptyHookFn,
onSecondaryOccursLock: emptyHookFn,
onPrewriteRow: emptyHookFn,
onTxnSuccess: emptyHookFn,
onTxnFailed: emptyHookFn,
}
}

22
vendor/github.com/pingcap/go-themis/util.go generated vendored Normal file
View file

@ -0,0 +1,22 @@
package themis
import "github.com/juju/errors"
func errorEqual(err1, err2 error) bool {
if err1 == err2 {
return true
}
e1 := errors.Cause(err1)
e2 := errors.Cause(err2)
if e1 == e2 {
return true
}
if e1 == nil || e2 == nil {
return e1 == e2
}
return e1.Error() == e2.Error()
}