Coverage for /private/tmp/im/impacket/impacket/ese.py : 68%

Hot-keys on this page
r m x p toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
# SECUREAUTH LABS. Copyright 2018 SecureAuth Corporation. All rights reserved. # # This software is provided under under a slightly modified version # of the Apache Software License. See the accompanying LICENSE file # for more information. # # Description: # Microsoft Extensive Storage Engine parser, just focused on trying # to parse NTDS.dit files (not meant as a full parser, although it might work) # # Author: # Alberto Solino (@agsolino) # # Reference for: # Structure. # # Excellent reference done by Joachim Metz # http://forensic-proof.com/wp-content/uploads/2011/07/Extensible-Storage-Engine-ESE-Database-File-EDB-format.pdf # # ToDo: # [ ] Parse multi-values properly # [ ] Support long values properly except: try: from ordereddict.ordereddict import OrderedDict except: from ordereddict import OrderedDict
# Constants
# Database state
# Page Flags
# Tag Flags
# Fixed Page Numbers
# Fixed FatherDataPages
# Catalog Types
# Column Types
JET_coltypNil : 'NULL', JET_coltypBit : 'Boolean', JET_coltypUnsignedByte : 'Signed byte', JET_coltypShort : 'Signed short', JET_coltypLong : 'Signed long', JET_coltypCurrency : 'Currency', JET_coltypIEEESingle : 'Single precision FP', JET_coltypIEEEDouble : 'Double precision FP', JET_coltypDateTime : 'DateTime', JET_coltypBinary : 'Binary', JET_coltypText : 'Text', JET_coltypLongBinary : 'Long Binary', JET_coltypLongText : 'Long Text', JET_coltypSLV : 'Obsolete', JET_coltypUnsignedLong : 'Unsigned long', JET_coltypLongLong : 'Long long', JET_coltypGUID : 'GUID', JET_coltypUnsignedShort: 'Unsigned short', JET_coltypMax : 'Max', }
JET_coltypNil : None, JET_coltypBit : (1,'B'), JET_coltypUnsignedByte : (1,'B'), JET_coltypShort : (2,'<h'), JET_coltypLong : (4,'<l'), JET_coltypCurrency : (8,'<Q'), JET_coltypIEEESingle : (4,'<f'), JET_coltypIEEEDouble : (8,'<d'), JET_coltypDateTime : (8,'<Q'), JET_coltypBinary : None, JET_coltypText : None, JET_coltypLongBinary : None, JET_coltypLongText : None, JET_coltypSLV : None, JET_coltypUnsignedLong : (4,'<L'), JET_coltypLongLong : (8,'<Q'), JET_coltypGUID : (16,'16s'), JET_coltypUnsignedShort: (2,'<H'), JET_coltypMax : None, }
# Tagged Data Type Flags
# Code pages
CODEPAGE_UNICODE : 'utf-16le', CODEPAGE_ASCII : 'ascii', CODEPAGE_WESTERN : 'cp1252', }
# Structures
'TableData' : b'', 'FatherDataPageNumber': 0, 'CurrentPageData' : b'', 'CurrentTag' : 0, }
('Random','<L=0'), ('CreationTime','<Q=0'), ('NetBiosName','16s=b""'), )
('CheckSum','<L=0'), ('Signature','"\xef\xcd\xab\x89'), ('Version','<L=0'), ('FileType','<L=0'), ('DBTime','<Q=0'), ('DBSignature',':',ESENT_JET_SIGNATURE), ('DBState','<L=0'), ('ConsistentPosition','<Q=0'), ('ConsistentTime','<Q=0'), ('AttachTime','<Q=0'), ('AttachPosition','<Q=0'), ('DetachTime','<Q=0'), ('DetachPosition','<Q=0'), ('LogSignature',':',ESENT_JET_SIGNATURE), ('Unknown','<L=0'), ('PreviousBackup','24s=b""'), ('PreviousIncBackup','24s=b""'), ('CurrentFullBackup','24s=b""'), ('ShadowingDisables','<L=0'), ('LastObjectID','<L=0'), ('WindowsMajorVersion','<L=0'), ('WindowsMinorVersion','<L=0'), ('WindowsBuildNumber','<L=0'), ('WindowsServicePackNumber','<L=0'), ('FileFormatRevision','<L=0'), ('PageSize','<L=0'), ('RepairCount','<L=0'), ('RepairTime','<Q=0'), ('Unknown2','28s=b""'), ('ScrubTime','<Q=0'), ('RequiredLog','<Q=0'), ('UpgradeExchangeFormat','<L=0'), ('UpgradeFreePages','<L=0'), ('UpgradeSpaceMapPages','<L=0'), ('CurrentShadowBackup','24s=b""'), ('CreationFileFormatVersion','<L=0'), ('CreationFileFormatRevision','<L=0'), ('Unknown3','16s=b""'), ('OldRepairCount','<L=0'), ('ECCCount','<L=0'), ('LastECCTime','<Q=0'), ('OldECCFixSuccessCount','<L=0'), ('ECCFixErrorCount','<L=0'), ('LastECCFixErrorTime','<Q=0'), ('OldECCFixErrorCount','<L=0'), ('BadCheckSumErrorCount','<L=0'), ('LastBadCheckSumTime','<Q=0'), ('OldCheckSumErrorCount','<L=0'), ('CommittedLog','<L=0'), ('PreviousShadowCopy','24s=b""'), ('PreviousDifferentialBackup','24s=b""'), ('Unknown4','40s=b""'), ('NLSMajorVersion','<L=0'), ('NLSMinorVersion','<L=0'), ('Unknown5','148s=b""'), ('UnknownFlags','<L=0'), )
('CheckSum','<L=0'), ('PageNumber','<L=0'), ) ('CheckSum','<L=0'), ('ECCCheckSum','<L=0'), ) ('CheckSum','<Q=0'), ) ('LastModificationTime','<Q=0'), ('PreviousPageNumber','<L=0'), ('NextPageNumber','<L=0'), ('FatherDataPage','<L=0'), ('AvailableDataSize','<H=0'), ('AvailableUncommittedDataSize','<H=0'), ('FirstAvailableDataOffset','<H=0'), ('FirstAvailablePageTag','<H=0'), ('PageFlags','<L=0'), ) ('ExtendedCheckSum1','<Q=0'), ('ExtendedCheckSum2','<Q=0'), ('ExtendedCheckSum3','<Q=0'), ('PageNumber','<Q=0'), ('Unknown','<Q=0'), ) # For sure the old format self.structure = self.structure_2003_SP0 + self.common # Exchange 2003 SP1 and Windows Vista and later self.structure = self.structure_0x620_0x0b + self.common else: # Windows 7 and later self.structure += self.extended_win7
('InitialNumberOfPages','<L=0'), ('ParentFatherDataPage','<L=0'), ('ExtentSpace','<L=0'), ('SpaceTreePageNumber','<L=0'), )
('CommonPageKey',':'), )
('CommonPageKeySize','<H=0'), ) ('LocalPageKeySize','<H=0'), ('_LocalPageKey','_-LocalPageKey','self["LocalPageKeySize"]'), ('LocalPageKey',':'), ('ChildPageNumber','<L=0'), ) # Include the common header
('CommonPageKey',':'), )
('CommonPageKeySize','<H=0'), ) ('LocalPageKeySize','<H=0'), ('_LocalPageKey','_-LocalPageKey','self["LocalPageKeySize"]'), ('LocalPageKey',':'), ('EntryData',':'), ) # Include the common header
('Unknown','<Q=0'), )
('PageKeySize','<H=0'), ('LastPageNumber','<L=0'), ('NumberOfPages','<L=0'), )
('RecordPageKey',':'), )
('LastFixedSize','<B=0'), ('LastVariableDataType','<B=0'), ('VariableSizeOffset','<H=0'), )
('FatherDataPageID','<L=0'), ('Type','<H=0'), ('Identifier','<L=0'), )
('ColumnType','<L=0'), ('SpaceUsage','<L=0'), ('ColumnFlags','<L=0'), ('CodePage','<L=0'), )
('FatherDataPageNumber','<L=0'), )
('SpaceUsage','<L=0'), # ('TableFlags','<L=0'), # ('InitialNumberOfPages','<L=0'), )
('SpaceUsage','<L=0'), ('IndexFlags','<L=0'), ('Locale','<L=0'), )
('SpaceUsage','<L=0'), # ('LVFlags','<L=0'), # ('InitialNumberOfPages','<L=0'), ) # ('RootFlag','<B=0'), # ('RecordOffset','<H=0'), # ('LCMapFlags','<L=0'), # ('KeyMost','<H=0'), ('Trailing',':'), )
# Depending on the type of data we'll end up building a different struct
elif dataType == CATALOG_TYPE_CALLBACK: raise Exception('CallBack types not supported!') else: LOG.error('Unknown catalog type 0x%x' % dataType) self.structure = () Structure.__init__(self,data)
#def pretty_print(x): # if x in '0123456789abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ!"#$%&\'()*+,-./:;<=>?@[\\]^_`{|}~ ': # return x # else: # return '.' # #def hexdump(data): # x=str(data) # strLen = len(x) # i = 0 # while i < strLen: # print "%04x " % i, # for j in range(16): # if i+j < strLen: # print "%02X" % ord(x[i+j]), # # else: # print " ", # if j%16 == 7: # print "", # print " ", # print ''.join(pretty_print(x) for x in x[i:i+16] ) # i += 16
flags = self.record['PageFlags'] if flags & FLAGS_EMPTY: print("\tEmpty") if flags & FLAGS_INDEX: print("\tIndex") if flags & FLAGS_LEAF: print("\tLeaf") else: print("\tBranch") if flags & FLAGS_LONG_VALUE: print("\tLong Value") if flags & FLAGS_NEW_CHECKSUM: print("\tNew Checksum") if flags & FLAGS_NEW_FORMAT: print("\tNew Format") if flags & FLAGS_PARENT: print("\tParent") if flags & FLAGS_ROOT: print("\tRoot") if flags & FLAGS_SPACE_TREE: print("\tSpace Tree")
baseOffset = len(self.record) self.record.dump() tags = self.data[-4*self.record['FirstAvailablePageTag']:]
print("FLAGS: ") self.printFlags()
print()
for i in range(self.record['FirstAvailablePageTag']): tag = tags[-4:] if self.__DBHeader['Version'] == 0x620 and self.__DBHeader['FileFormatRevision'] > 11 and self.__DBHeader['PageSize'] > 8192: valueSize = unpack('<H', tag[:2])[0] & 0x7fff valueOffset = unpack('<H',tag[2:])[0] & 0x7fff hexdump((self.data[baseOffset+valueOffset:][:6])) pageFlags = ord(self.data[baseOffset+valueOffset:][1]) >> 5 #print "TAG FLAG: 0x%x " % (unpack('<L', self.data[baseOffset+valueOffset:][:4]) ) >> 5 #print "TAG FLAG: 0x " , ord(self.data[baseOffset+valueOffset:][0]) else: valueSize = unpack('<H', tag[:2])[0] & 0x1fff pageFlags = (unpack('<H', tag[2:])[0] & 0xe000) >> 13 valueOffset = unpack('<H',tag[2:])[0] & 0x1fff
print("TAG %-8d offset:0x%-6x flags:0x%-4x valueSize:0x%x" % (i,valueOffset,pageFlags,valueSize)) #hexdump(self.getTag(i)[1]) tags = tags[:-4]
if self.record['PageFlags'] & FLAGS_ROOT > 0: rootHeader = ESENT_ROOT_HEADER(self.getTag(0)[1]) rootHeader.dump() elif self.record['PageFlags'] & FLAGS_LEAF == 0: # Branch Header flags, data = self.getTag(0) branchHeader = ESENT_BRANCH_HEADER(data) branchHeader.dump() else: # Leaf Header flags, data = self.getTag(0) if self.record['PageFlags'] & FLAGS_SPACE_TREE > 0: # Space Tree spaceTreeHeader = ESENT_SPACE_TREE_HEADER(data) spaceTreeHeader.dump() else: leafHeader = ESENT_LEAF_HEADER(data) leafHeader.dump()
# Print the leaf/branch tags for tagNum in range(1,self.record['FirstAvailablePageTag']): flags, data = self.getTag(tagNum) if self.record['PageFlags'] & FLAGS_LEAF == 0: # Branch page branchEntry = ESENT_BRANCH_ENTRY(flags, data) branchEntry.dump() elif self.record['PageFlags'] & FLAGS_LEAF > 0: # Leaf page if self.record['PageFlags'] & FLAGS_SPACE_TREE > 0: # Space Tree spaceTreeEntry = ESENT_SPACE_TREE_ENTRY(data) #spaceTreeEntry.dump()
elif self.record['PageFlags'] & FLAGS_INDEX > 0: # Index Entry indexEntry = ESENT_INDEX_ENTRY(data) #indexEntry.dump() elif self.record['PageFlags'] & FLAGS_LONG_VALUE > 0: # Long Page Value raise Exception('Long value still not supported') else: # Table Value leafEntry = ESENT_LEAF_ENTRY(flags, data) dataDefinitionHeader = ESENT_DATA_DEFINITION_HEADER(leafEntry['EntryData']) dataDefinitionHeader.dump() catalogEntry = ESENT_CATALOG_DATA_DEFINITION_ENTRY(leafEntry['EntryData'][len(dataDefinitionHeader):]) catalogEntry.dump() hexdump(leafEntry['EntryData'])
raise Exception('Trying to grab an unknown tag 0x%x' % tagNum)
valueSize = unpack('<H', tag[:2])[0] & 0x7fff valueOffset = unpack('<H',tag[2:])[0] & 0x7fff tmpData = list(self.data[baseOffset+valueOffset:][:valueSize]) pageFlags = ord(tmpData[1]) >> 5 tmpData[1] = chr(ord(tmpData[1:2]) & 0x1f) tagData = "".join(tmpData) else:
#return pageFlags, self.data[baseOffset+valueOffset:][:valueSize]
else: self.__DB = open(self.__fileName,"rb")
indent = ' '
print("Database version: 0x%x, 0x%x" % (self.__DBHeader['Version'], self.__DBHeader['FileFormatRevision'] )) print("Page size: %d " % self.__pageSize) print("Number of pages: %d" % self.__totalPages) print() print("Catalog for %s" % self.__fileName) for table in list(self.__tables.keys()): print("[%s]" % table.decode('utf8')) print("%sColumns " % indent) for column in list(self.__tables[table]['Columns'].keys()): record = self.__tables[table]['Columns'][column]['Record'] print("%s%-5d%-30s%s" % (indent*2, record['Identifier'], column.decode('utf-8'),ColumnTypeToName[record['ColumnType']])) print("%sIndexes"% indent) for index in list(self.__tables[table]['Indexes'].keys()): print("%s%s" % (indent*2, index.decode('utf-8'))) print("")
else: raise Exception('Unknown type 0x%x' % catalogEntry['Type'])
else: numEntries = dataDefinitionHeader['LastVariableDataType']
# Print the leaf/branch tags # Leaf page pass pass pass else: # Table Value
# Parse all the pages starting at pageNum and commit table data
# Branch page
LOG.debug("Reading Boot Sector for %s" % self.__volumeName)
remaining = self.__pageSize - len(data) data += self.__DB.read(remaining) # Special case for the first page else:
# Returns a cursos for later use
# Let's position the cursor at the leaf levels for fast reading # There are no records done = True # Branch page, move on to the next page else:
else: return None
# No more data in this page, chau
# Leaf page raise Exception('FLAGS_SPACE_TREE > 0') raise Exception('FLAGS_INDEX > 0') raise Exception('FLAGS_LONG_VALUE > 0') else: # Table Value
return None
#hexdump(tag)
# No more tags in this page, search for the next one on the right # No more pages, chau else: else:
# So my brain doesn't forget, the data record is composed of: # Header # Fixed Size Data (ID < 127) # The easiest to parse. Their size is fixed in the record. You can get its size # from the Column Record, field SpaceUsage # Variable Size Data (127 < ID < 255) # At VariableSizeOffset you get an array of two bytes per variable entry, pointing # to the length of the value. Values start at: # numEntries = LastVariableDataType - 127 # VariableSizeOffset + numEntries * 2 (bytes) # Tagged Data ( > 255 ) # After the Variable Size Value, there's more data for the tagged values. # Right at the beginning there's another array (taggedItems), pointing to the # values, size. # # The interesting thing about this DB records is there's no need for all the columns to be there, hence # saving space. That's why I got over all the columns, and if I find data (of any type), i assign it. If # not, the column's empty. # # There are a lot of caveats in the code, so take your time to explore it. # # ToDo: Better complete this description #
#dataDefinitionHeader.dump()
#columnRecord.dump() # Fixed Size column data type, still available data
# Variable data type index = columnRecord['Identifier'] - 127 - 1 itemLen = unpack('<H',tag[variableSizeOffset+index*2:][:2])[0]
if itemLen & 0x8000: # Empty item itemLen = prevItemLen record[column] = None else: itemValue = tag[variableSizeOffset+variableDataBytesProcessed:][:itemLen-prevItemLen] record[column] = itemValue
#if columnRecord['Identifier'] <= dataDefinitionHeader['LastVariableDataType']: variableDataBytesProcessed +=itemLen-prevItemLen
prevItemLen = itemLen
# Have we parsed the tagged items already? #hexdump(tag[index:]) # As of Windows 7 and later ( version 0x620 revision 0x11) the # tagged data type flags are always present flagsPresent = 1 else: #print "ID: %d, Offset:%d, firstOffset:%d, index:%d, flag: 0x%x" % (taggedIdentifier, taggedOffset,firstOffsetTag,index, flagsPresent) # We reached the end of the variable size array
# Calculate length of variable items # Ugly.. should be redone #print "ID: %d, Offset: %d, Len: %d, flags: %d" % (prevKey, offset0, offset-offset0, flags)
# Tagged data type # If item have flags, we should skip them else:
#print "ID: %d, itemFlag: 0x%x" %( columnRecord['Identifier'], itemFlag) LOG.error('Unsupported tag column: %s, flag:0x%x' % (column, itemFlag)) record[column] = None # ToDo: Parse multi-values properly else:
else: else:
# If we understand the data type, we unpack it and cast it accordingly # otherwise, we just encode it in hex # A multi value data, we won't decode it, just leave it this way # Let's handle strings raise Exception('Unknown codepage 0x%x'% columnRecord['CodePage'])
except Exception: LOG.debug("Exception:", exc_info=True) LOG.debug('Fixing Record[%r][%d]: %r' % (column, columnRecord['ColumnType'], record[column])) record[column] = record[column].decode(stringDecoder, "replace") pass else: else:
|