ceph pg incomplete: rbd image-format 2 data recovery

/*
 * old-style rbd image 'foo' consists of objects
 *   foo.rbd      - image metadata
 *   rb.<idhi>.<idlo>.00000000
 *   rb.<idhi>.<idlo>.00000001
 *   ...          - data
 */

#define RBD_SUFFIX	 	".rbd"
#define RBD_DIRECTORY           "rbd_directory"
#define RBD_INFO                "rbd_info"

/* New-style rbd image 'foo' consists of objects
 *   rbd_id.foo              - id of image
 *   rbd_header.<id>         - image metadata
 *   rbd_data.<id>.00000000
 *   rbd_data.<id>.00000001
 *   ...                     - data
 */

#define RBD_HEADER_PREFIX      "rbd_header."
#define RBD_DATA_PREFIX        "rbd_data."
#define RBD_ID_PREFIX          "rbd_id."

# rbd -p images ls
08909734-66fa-48e3-ab5e-2e2b8bb3a58c

# rbd -p images info 08909734-66fa-48e3-ab5e-2e2b8bb3a58c
rbd image '08909734-66fa-48e3-ab5e-2e2b8bb3a58c':
    size 810 MB in 102 objects
    order 23 (8192 kB objects)
    block_name_prefix: rbd_data.5a57484353d0cd
    format: 2
    features: layering

# cat rbd_id.08909734-66fa-48e3-ab5e-2e2b8bb3a58c
5a57484353d0cd

# rados listomapvals -p images rbd_header.5a57484353d0cd
features
value: (8 bytes) :
0000 : 01 00 00 00 00 00 00 00                         : ........
object_prefix
value: (27 bytes) :
0000 : 17 00 00 00 72 62 64 5f 64 61 74 61 2e 35 61 35 : ....rbd_data.5a5
0010 : 37 34 38 34 33 35 33 64 30 63 64                : 7484353d0cd
order
value: (1 bytes) :
0000 : 17                                              : .
size
value: (8 bytes) :
0000 : 00 8c a3 32 00 00 00 00                         : ...2....
snap_seq
value: (8 bytes) :
0000 : 00 00 00 00 00 00 00 00                         : ........

# ceph osd dump
...
pool 35 'images' replicated size 2 min_size 1 crush_ruleset 0 object_hash rjenkins pg_num 512 pgp_num 512 last_change 46681 flags hashpspool stripe_width 0
...

# rbd -p images info 08909734-66fa-48e3-ab5e-2e2b8bb3a58c
rbd image '08909734-66fa-48e3-ab5e-2e2b8bb3a58c':
    size 810 MB in 102 objects
    order 23 (8192 kB objects)
    block_name_prefix: rbd_data.5a57484353d0cd
    format: 2
    features: layering

# ceph osd map images rbd_data.b0e882ae8944a.0000000000000134
osdmap e53074 pool 'images' (35) object 'rbd_data.b0e882ae8944a.0000000000000134' -> pg 35.9f6e4c67 (35.67) -> up ([3,11], p3) acting ([3,11], p3)

'''Implements a straight Jenkins lookup hash - http://burtleburtle.net/bob/hash/doobs.html

Usage: 
    from jhash import jhash
    print jhash('My hovercraft is full of eels')

Returns: unsigned 32 bit integer value

Prereqs: None

Tested with Python 2.6.
Version 1.00 - der@dod.no - 23.08.2010

Partly based on the Perl module Digest::JHash
http://search.cpan.org/~shlomif/Digest-JHash-0.06/lib/Digest/JHash.pm

Original copyright notice:
    By Bob Jenkins, 1996.  bob_jenkins@burtleburtle.net.  You may use this
    code any way you wish, private, educational, or commercial.  It's free.

    See http://burtleburtle.net/bob/hash/evahash.html
    Use for hash table lookup, or anything where one collision in 2^^32 is
    acceptable.  Do NOT use for cryptographic purposes.
'''

def mix(a, b, c):
    '''mix() -- mix 3 32-bit values reversibly.
For every delta with one or two bits set, and the deltas of all three
  high bits or all three low bits, whether the original value of a,b,c
  is almost all zero or is uniformly distributed,
* If mix() is run forward or backward, at least 32 bits in a,b,c
  have at least 1/4 probability of changing.
* If mix() is run forward, every bit of c will change between 1/3 and
  2/3 of the time.  (Well, 22/100 and 78/100 for some 2-bit deltas.)'''
    # Need to constrain U32 to only 32 bits using the & 0xffffffff 
    # since Python has no native notion of integers limited to 32 bit
    # http://docs.python.org/library/stdtypes.html#numeric-types-int-float-long-complex
    a &= 0xffffffff; b &= 0xffffffff; c &= 0xffffffff
    a -= b; a -= c; a ^= (c>>13); a &= 0xffffffff
    b -= c; b -= a; b ^= (a<<8); b &= 0xffffffff
    c -= a; c -= b; c ^= (b>>13); c &= 0xffffffff
    a -= b; a -= c; a ^= (c>>12); a &= 0xffffffff
    b -= c; b -= a; b ^= (a<<16); b &= 0xffffffff
    c -= a; c -= b; c ^= (b>>5); c &= 0xffffffff
    a -= b; a -= c; a ^= (c>>3); a &= 0xffffffff
    b -= c; b -= a; b ^= (a<<10); b &= 0xffffffff
    c -= a; c -= b; c ^= (b>>15); c &= 0xffffffff
    return a, b, c

def jhash(data, initval = 0):
    '''hash() -- hash a variable-length key into a 32-bit value
  data    : the key (the unaligned variable-length array of bytes)
  initval : can be any 4-byte value, defaults to 0
Returns a 32-bit value.  Every bit of the key affects every bit of
the return value.  Every 1-bit and 2-bit delta achieves avalanche.'''
    length = lenpos = len(data)

    # empty string returns 0
    if length == 0:
        return 0

    # Set up the internal state
    a = b = 0x9e3779b9 # the golden ratio; an arbitrary value
    c = initval        # the previous hash value
    p = 0              # string offset

    # ------------------------- handle most of the key in 12 byte chunks
    while lenpos >= 12:
        a += (ord(data[p+0]) + (ord(data[p+1])<<8) + (ord(data[p+2])<<16) + (ord(data[p+3])<<24))
        b += (ord(data[p+4]) + (ord(data[p+5])<<8) + (ord(data[p+6])<<16) + (ord(data[p+7])<<24))
        c += (ord(data[p+8]) + (ord(data[p+9])<<8) + (ord(data[p+10])<<16) + (ord(data[p+11])<<24))
        a, b, c = mix(a, b, c)
        p += 12
        lenpos -= 12

    # ------------------------- handle the last 11 bytes
    c += length
    if lenpos >= 11: c += ord(data[p+10])<<24
    if lenpos >= 10: c += ord(data[p+9])<<16
    if lenpos >= 9:  c += ord(data[p+8])<<8
    # the first byte of c is reserved for the length
    if lenpos >= 8:  b += ord(data[p+7])<<24
    if lenpos >= 7:  b += ord(data[p+6])<<16
    if lenpos >= 6:  b += ord(data[p+5])<<8
    if lenpos >= 5:  b += ord(data[p+4])
    if lenpos >= 4:  a += ord(data[p+3])<<24
    if lenpos >= 3:  a += ord(data[p+2])<<16
    if lenpos >= 2:  a += ord(data[p+1])<<8
    if lenpos >= 1:  a += ord(data[p+0])
    a, b, c = mix(a, b, c)

    # ------------------------- report the result
    return c

if __name__ == "__main__":
    hashstr = sys.argv[1]
    myhash = jhash(hashstr)
    myhash2 = myhash % int(sys.argv[2])
    print "%x" % myhash2

# ceph pg dump > /tmp/pgdump # just once

# cat /tmp/pgdump | grep ^35.67
dumped all in format plain
35.67	0	0	0	0	0	2	2	active+clean	2015-01-15 06:48:51.979426	46681	53074:6358	[3,11]	0	[3,11]	0	46681	2015-01-15 06:48:51.979318	46681	2015-01-14 06:47:33.392701

/var/lib/ceph/osd/ceph-3/current/35.67_head/DIR_7/DIR_6/DIR_C/rbd\udata.b0e882ae8944a.0000000000000134__head_9F6E4C67__23

#!/bin/sh
#
# AUTHORS
# Shawn Moore <smmoore@catawba.edu>
# Rodney Rymer <rrr@catawba.edu>
#
#
# REQUIREMENTS
# GNU Awk (gawk)
#
#
# NOTES
# This utility assumes one copy of all object files needed to construct the rbd
# are located in the present working direcory at the time of execution.  
# For example all the rb.0.1032.5e69c215.* files.
#
# When listing the "RBD_SIZE_IN_BYTES", be sure you list the full potential size, 
# not just what it appears to be. If you do not know the true size of the rbd,
# you can input a size in bytes that you know is larger than the disk could be
# and it will be a large sparse file with un-partioned space at the end of the
# disk.  In our tests, this doesn't occupy any more space/objects in the cluster
# but the rbd could be resized from within the rbd (VM) to grow.  Once you bring
# it up and are able to find the true size, you can resize with "rbd resize ..".
#
# To obtain needed utility input information if not already known run:
# rbd info RBD
#
# To find needed files we run the following command on all nodes that might have
# copies of the rbd objects:
# find /${CEPH} -type f -name rb.0.1032.5e69c215.*
# Then copy the files to a single location from all nodes.  If using btrfs be
# sure to pay attention to the btrfs snapshots that ceph takes on it's own.
# You may want the "current" or one of the "snaps".
#
# We are actually taking our own btrfs snapshots cluster osd wide at the same
# time with parallel ssh and then using "btrfs subvolume find-new" command to
# merge them all together for disaster recovery and also outside of ceph rbd
# versioning.
#
# Hopefully once the btrfs send/recv functionality is stable we can switch to it.
#
#
# This utility works for us but may not for you.  Always test with non-critical
# data first.
#

# Rados object size
obj_size=4194304

# DD bs value
rebuild_block_size=512

rbd="${1}"
base="${2}"
rbd_size="${3}"
if [ "${1}" = "-h" -o "${1}" = "--help" -o "${rbd}" = "" -o "${base}" = "" -o "${rbd_size}" = "" ]; then
  echo "USAGE: $(echo ${0} | awk -F/ '{print $NF}') RESTORE_RBD BLOCK_PREFIX RBD_SIZE_IN_BYTES"
  exit 1
fi
base_files=$(ls -1 ${base}.* 2>/dev/null | wc -l | awk '{print $1}')
if [ ${base_files} -lt 1 ]; then
  echo "COULD NOT FIND FILES FOR ${base} IN $(pwd)"
  exit
fi

# Create full size sparse image.  Could use truncate, but wanted
# as few required files and dd what a must.
dd if=/dev/zero of=${rbd} bs=1 count=0 seek=${rbd_size} 2>/dev/null

for file_name in $(ls -1 ${base}.* 2>/dev/null); do
  seek_loc=$(echo ${file_name} | awk -F_ '{print $1}' | awk -v os=${obj_size} -v rs=${rebuild_block_size} -F. '{print os*strtonum("0x" $NF)/rs}')
  dd conv=notrunc if=${file_name} of=${rbd} seek=${seek_loc} bs=${rebuild_block_size} 2>/dev/null
done

v="volume-uuid"
losetup /dev/loop0 $v
kpartx -a /dev/loop0
mkdir -p /mnt/$v
mount /dev/mapper/loop0p1 /mnt/$v