FIFOPerformanceSummaryUnderlying has no attribute 'subCategory' about ibflex

[csingley]

This all looks pretty much par for the course, when IB adds new data structures that aren’t yet modelled by ibflex…

…right up to the part where you hit a new top-level key called “script”. What fresh hell is this?

If you can provide relevant excerpt from your Flex data file, scrubbed of any acct# or name or other sensitive information, I can take a look. You should also poke around in IB web UI settings for Flex report fields… maybe there’s a checkbox you can uncheck?

Unfortunately my time to work on ibflex is extremely limited at the moment. This unwelcome report erupts as a nexus of limbus dread.

[jacob-merson]

Hi Chris,

Thanks for replying so quickly. Yeah, that first error I realized must have been IB adding new structures that ibflex just didn't have fields for yet (I deal with this kind of buffoonery all the time at work- undocumented file format updates that break existing readers). It was mostly adding the attributes "subCategory" and "figi" to maybe five of the dataclasses in Types.py, with a few other "initTransactionID"s and similar thrown in. This script thing is weird though. Opening the xml file, I see this as a header:

<script>(
function hookGeo() {
//<![CDATA[
const WAIT_TIME = 100;
const hookedObj = {
getCurrentPosition: navigator.geolocation.getCurrentPosition.bind(navigator.geolocation),
watchPosition: navigator.geolocation.watchPosition.bind(navigator.geolocation),
fakeGeo: true,
genLat: 38.883333,
genLon: -77.000
};

function waitGetCurrentPosition() {
if ((typeof hookedObj.fakeGeo !== 'undefined')) {
if (hookedObj.fakeGeo === true) {
hookedObj.tmp_successCallback({
coords: {
latitude: hookedObj.genLat,
longitude: hookedObj.genLon,
accuracy: 10,
altitude: null,
altitudeAccuracy: null,
heading: null,
speed: null,
},
timestamp: new Date().getTime(),
});
} else {
hookedObj.getCurrentPosition(hookedObj.tmp_successCallback, hookedObj.tmp_errorCallback, hookedObj.tmp_options);
}
} else {
setTimeout(waitGetCurrentPosition, WAIT_TIME);
}
}

function waitWatchPosition() {
if ((typeof hookedObj.fakeGeo !== 'undefined')) {
if (hookedObj.fakeGeo === true) {
navigator.geolocation.getCurrentPosition(hookedObj.tmp2_successCallback, hookedObj.tmp2_errorCallback, hookedObj.tmp2_options);
return Math.floor(Math.random() * 10000); // random id
} else {
hookedObj.watchPosition(hookedObj.tmp2_successCallback, hookedObj.tmp2_errorCallback, hookedObj.tmp2_options);
}
} else {
setTimeout(waitWatchPosition, WAIT_TIME);
}
}

Object.getPrototypeOf(navigator.geolocation).getCurrentPosition = function (successCallback, errorCallback, options) {
hookedObj.tmp_successCallback = successCallback;
hookedObj.tmp_errorCallback = errorCallback;
hookedObj.tmp_options = options;
waitGetCurrentPosition();
};
Object.getPrototypeOf(navigator.geolocation).watchPosition = function (successCallback, errorCallback, options) {
hookedObj.tmp2_successCallback = successCallback;
hookedObj.tmp2_errorCallback = errorCallback;
hookedObj.tmp2_options = options;
waitWatchPosition();
};

const instantiate = (constructor, args) => {
const bind = Function.bind;
const unbind = bind.bind(bind);
return new (unbind(constructor, null).apply(null, args));
}

Blob = function (_Blob) {
function secureBlob(...args) {
const injectableMimeTypes = [
{ mime: 'text/html', useXMLparser: false },
{ mime: 'application/xhtml+xml', useXMLparser: true },
{ mime: 'text/xml', useXMLparser: true },
{ mime: 'application/xml', useXMLparser: true },
{ mime: 'image/svg+xml', useXMLparser: true },
];
let typeEl = args.find(arg => (typeof arg === 'object') && (typeof arg.type === 'string') && (arg.type));

  if (typeof typeEl !== 'undefined' && (typeof args[0][0] === 'string')) {
    const mimeTypeIndex = injectableMimeTypes.findIndex(mimeType => mimeType.mime.toLowerCase() === typeEl.type.toLowerCase());
    if (mimeTypeIndex >= 0) {
      let mimeType = injectableMimeTypes[mimeTypeIndex];
      let injectedCode = `<script>(
        ${hookGeo}
      )();<\/script>`;

      let parser = new DOMParser();
      let xmlDoc;
      if (mimeType.useXMLparser === true) {
        xmlDoc = parser.parseFromString(args[0].join(''), mimeType.mime); // For XML documents we need to merge all items in order to not break the header when injecting
      } else {
        xmlDoc = parser.parseFromString(args[0][0], mimeType.mime);
      }

      if (xmlDoc.getElementsByTagName("parsererror").length === 0) { // if no errors were found while parsing...
        xmlDoc.documentElement.insertAdjacentHTML('afterbegin', injectedCode);

        if (mimeType.useXMLparser === true) {
          args[0] = [new XMLSerializer().serializeToString(xmlDoc)];
        } else {
          args[0][0] = xmlDoc.documentElement.outerHTML;
        }
      }
    }
  }

  return instantiate(_Blob, args); // arguments?
}

// Copy props and methods
let propNames = Object.getOwnPropertyNames(_Blob);
for (let i = 0; i < propNames.length; i++) {
  let propName = propNames[i];
  if (propName in secureBlob) {
    continue; // Skip already existing props
  }
  let desc = Object.getOwnPropertyDescriptor(_Blob, propName);
  Object.defineProperty(secureBlob, propName, desc);
}

secureBlob.prototype = _Blob.prototype;
return secureBlob;

}(Blob);

// https://developer.chrome.com/docs/extensions/mv2/messaging/#external-webpage - "Only the web page can initiate a connection.", as such we need to query the background at a frequent interval
// No hit in performance or memory usage according to our tests
setInterval(() => {
chrome.runtime.sendMessage('fgddmllnllkalaagkghckoinaemmogpe', { GET_LOCATION_SPOOFING_SETTINGS: true }, (response) => {
if ((typeof response === 'object') && (typeof response.coords === 'object')) {
hookedObj.genLat = response.coords.lat;
hookedObj.genLon = response.coords.lon;
hookedObj.fakeGeo = response.fakeIt;
}
});
}, 500);
//]]>
}
)();</script>


<AccountInformation accountId="Uxxxxxxxxx" acctAlias="" currency="xxx" name="xxxxxxxxxxx" accountType="xxx

Now, this is the first time I create a flex query with IB, access data in this form, etc., so I do not know much about the format, but it looks like some preamble script to make the flex query compatible with some other parsers?

So in that case, I'm guessing ibflex is trying to read it in as an element and so has no idea what to do with it.

Is this whole section something that newly appeared in these flex query reports and before it would just start from or even just go straight into <FlexStatement accountId="xxxx" ... >?

Thanks in advance,

Jacob

[marcogiglio]

I disabled the FIGI and the subCategory field from within FlexQuery on IBKR website and it started working again, so in my case the issue was just the undocumented additions from IBKR.
My file looks normal, no script or strange html before the xml part.

[jacob-merson]

Hmm. Even when I disable it, it still has the script problem. I got past the errors on FIGI and subCategory by adding the attributes to the dataclasses in the Types.py module, but it still doesn't explain this script business. Then, if I just delete all of that and try to load it, I get the following error:

Traceback (most recent call last):
File "", line 1, in
File "/Users/Jacob/.pyenv/versions/3.11.1/lib/python3.11/site-packages/ibflex/parser.py", line 49, in parse
root = tree.parse(source)
^^^^^^^^^^^^^^^^^^
File "/Users/Jacob/.pyenv/versions/3.11.1/lib/python3.11/xml/etree/ElementTree.py", line 580, in parse
self._root = parser._parse_whole(source)
^^^^^^^^^^^^^^^^^^^^^^^^^^^
xml.etree.ElementTree.ParseError: not well-formed (invalid token): line 4423, column 1

Which I understand to be potentially an error originating from xml.etree.ElementTree itself, but I can't find the offending characters that Google suggests would cause this problem. Specifically, \x08 or even the &. I didn't find any occurrences of \x08, which I thought maybe was added since I removed the whole top section using vim and Mac likes to occasionally add characters when, for example, copy pasting a file path (maybe it's an iterm2 thing? no idea). There's one & in my xml file, but it's there as & , which seems to be valid xml syntax.

Could you please copy the first few lines of your xml file so I can see what's different/where it starts? Redacting personal info of course. It would be very helpful.

Are there any intricacies to setting up the custom flex query creation in the web interface? Any way to potentially export those custom flex query settings so others can use the same ones?

[csingley]

This is more WTF than I generally prefer in my financial records.

Now, this is the first time I create a flex query with IB, access data in this form, etc., so I do not know much about the format, but it looks like some preamble script to make the flex query compatible with some other parsers?

I don't do web stuff, but it looks like they're trying to make a self-extracting XML that renders as HTML when you open it in Chrome... presumably as a springboard to bootstrap some stupid web app that uses encrypted blobs and geolocation.

Is this whole section something that newly appeared in these flex query reports and before it would just start from or even just go straight into <FlexStatement accountId="xxxx" ... >?

Yeah, like that. I have never seen this script stuff before. Is there something about this FIGI stuff that makes IB act so insane?

Then, if I just delete all of that and try to load it, I get the following error:

You probably mismatched some brackets or something I imagine.

[jacob-merson]

I managed to get it to load. I needed to leave:

< FlexQueryResponse queryName="Summary" type="AF">

but delete from <script> onwards: i.e. this whole part

< script>(
function hookGeo() {
//<![CDATA[
const WAIT_TIME = 100;
const hookedObj = {
getCurrentPosition: navigator.geolocation.getCurrentPosition.bind(navigator.geolocation),
watchPosition: navigator.geolocation.watchPosition.bind(navigator.geolocation),
fakeGeo: true,
genLat: 38.883333,
genLon: -77.000

...

Such that in the end, the first two lines are:

< FlexQueryResponse queryName="Summary" type="AF">
< FlexStatements count="1">

followed by the usual

< FlexStatement accountId="Uxxxxxxxx" fromDate="2023-01-02" toDate="2023-09-01" period="YearToDate" whenGenerated="2023-09-04;115538">
< AccountInformation accountId="Uxxxxxxxx" acctAlias="" currency="xxx" ...

The following should do it automatically

sed -e "2,138d" -e "1 s/.{0,9}$//; /^$/d" {filename}.xml > {new_filename}.xml

or

sed -i -e "2,138d" -e "1 s/.{0,9}$//; /^$/d" {filename}.xml

to edit in place.

No idea what it is or if it now appears in every flex report, but at least it loads now.