Comments (6)
This all looks pretty much par for the course, when IB adds new data structures that aren’t yet modelled by ibflex…
…right up to the part where you hit a new top-level key called “script”. What fresh hell is this?
If you can provide relevant excerpt from your Flex data file, scrubbed of any acct# or name or other sensitive information, I can take a look. You should also poke around in IB web UI settings for Flex report fields… maybe there’s a checkbox you can uncheck?
Unfortunately my time to work on ibflex is extremely limited at the moment. This unwelcome report erupts as a nexus of limbus dread.
from ibflex.
Hi Chris,
Thanks for replying so quickly. Yeah, that first error I realized must have been IB adding new structures that ibflex just didn't have fields for yet (I deal with this kind of buffoonery all the time at work- undocumented file format updates that break existing readers). It was mostly adding the attributes "subCategory" and "figi" to maybe five of the dataclasses in Types.py, with a few other "initTransactionID"s and similar thrown in. This script thing is weird though. Opening the xml file, I see this as a header:
<script>(
function hookGeo() {
//<![CDATA[
const WAIT_TIME = 100;
const hookedObj = {
getCurrentPosition: navigator.geolocation.getCurrentPosition.bind(navigator.geolocation),
watchPosition: navigator.geolocation.watchPosition.bind(navigator.geolocation),
fakeGeo: true,
genLat: 38.883333,
genLon: -77.000
};
function waitGetCurrentPosition() {
if ((typeof hookedObj.fakeGeo !== 'undefined')) {
if (hookedObj.fakeGeo === true) {
hookedObj.tmp_successCallback({
coords: {
latitude: hookedObj.genLat,
longitude: hookedObj.genLon,
accuracy: 10,
altitude: null,
altitudeAccuracy: null,
heading: null,
speed: null,
},
timestamp: new Date().getTime(),
});
} else {
hookedObj.getCurrentPosition(hookedObj.tmp_successCallback, hookedObj.tmp_errorCallback, hookedObj.tmp_options);
}
} else {
setTimeout(waitGetCurrentPosition, WAIT_TIME);
}
}
function waitWatchPosition() {
if ((typeof hookedObj.fakeGeo !== 'undefined')) {
if (hookedObj.fakeGeo === true) {
navigator.geolocation.getCurrentPosition(hookedObj.tmp2_successCallback, hookedObj.tmp2_errorCallback, hookedObj.tmp2_options);
return Math.floor(Math.random() * 10000); // random id
} else {
hookedObj.watchPosition(hookedObj.tmp2_successCallback, hookedObj.tmp2_errorCallback, hookedObj.tmp2_options);
}
} else {
setTimeout(waitWatchPosition, WAIT_TIME);
}
}
Object.getPrototypeOf(navigator.geolocation).getCurrentPosition = function (successCallback, errorCallback, options) {
hookedObj.tmp_successCallback = successCallback;
hookedObj.tmp_errorCallback = errorCallback;
hookedObj.tmp_options = options;
waitGetCurrentPosition();
};
Object.getPrototypeOf(navigator.geolocation).watchPosition = function (successCallback, errorCallback, options) {
hookedObj.tmp2_successCallback = successCallback;
hookedObj.tmp2_errorCallback = errorCallback;
hookedObj.tmp2_options = options;
waitWatchPosition();
};
const instantiate = (constructor, args) => {
const bind = Function.bind;
const unbind = bind.bind(bind);
return new (unbind(constructor, null).apply(null, args));
}
Blob = function (_Blob) {
function secureBlob(...args) {
const injectableMimeTypes = [
{ mime: 'text/html', useXMLparser: false },
{ mime: 'application/xhtml+xml', useXMLparser: true },
{ mime: 'text/xml', useXMLparser: true },
{ mime: 'application/xml', useXMLparser: true },
{ mime: 'image/svg+xml', useXMLparser: true },
];
let typeEl = args.find(arg => (typeof arg === 'object') && (typeof arg.type === 'string') && (arg.type));
if (typeof typeEl !== 'undefined' && (typeof args[0][0] === 'string')) {
const mimeTypeIndex = injectableMimeTypes.findIndex(mimeType => mimeType.mime.toLowerCase() === typeEl.type.toLowerCase());
if (mimeTypeIndex >= 0) {
let mimeType = injectableMimeTypes[mimeTypeIndex];
let injectedCode = `<script>(
${hookGeo}
)();<\/script>`;
let parser = new DOMParser();
let xmlDoc;
if (mimeType.useXMLparser === true) {
xmlDoc = parser.parseFromString(args[0].join(''), mimeType.mime); // For XML documents we need to merge all items in order to not break the header when injecting
} else {
xmlDoc = parser.parseFromString(args[0][0], mimeType.mime);
}
if (xmlDoc.getElementsByTagName("parsererror").length === 0) { // if no errors were found while parsing...
xmlDoc.documentElement.insertAdjacentHTML('afterbegin', injectedCode);
if (mimeType.useXMLparser === true) {
args[0] = [new XMLSerializer().serializeToString(xmlDoc)];
} else {
args[0][0] = xmlDoc.documentElement.outerHTML;
}
}
}
}
return instantiate(_Blob, args); // arguments?
}
// Copy props and methods
let propNames = Object.getOwnPropertyNames(_Blob);
for (let i = 0; i < propNames.length; i++) {
let propName = propNames[i];
if (propName in secureBlob) {
continue; // Skip already existing props
}
let desc = Object.getOwnPropertyDescriptor(_Blob, propName);
Object.defineProperty(secureBlob, propName, desc);
}
secureBlob.prototype = _Blob.prototype;
return secureBlob;
}(Blob);
// https://developer.chrome.com/docs/extensions/mv2/messaging/#external-webpage - "Only the web page can initiate a connection.", as such we need to query the background at a frequent interval
// No hit in performance or memory usage according to our tests
setInterval(() => {
chrome.runtime.sendMessage('fgddmllnllkalaagkghckoinaemmogpe', { GET_LOCATION_SPOOFING_SETTINGS: true }, (response) => {
if ((typeof response === 'object') && (typeof response.coords === 'object')) {
hookedObj.genLat = response.coords.lat;
hookedObj.genLon = response.coords.lon;
hookedObj.fakeGeo = response.fakeIt;
}
});
}, 500);
//]]>
}
)();</script>
<AccountInformation accountId="Uxxxxxxxxx" acctAlias="" currency="xxx" name="xxxxxxxxxxx" accountType="xxx
Now, this is the first time I create a flex query with IB, access data in this form, etc., so I do not know much about the format, but it looks like some preamble script to make the flex query compatible with some other parsers?
So in that case, I'm guessing ibflex is trying to read it in as an element and so has no idea what to do with it.
Is this whole section something that newly appeared in these flex query reports and before it would just start from or even just go straight into <FlexStatement accountId="xxxx" ... >?
Thanks in advance,
Jacob
from ibflex.
I disabled the FIGI and the subCategory field from within FlexQuery on IBKR website and it started working again, so in my case the issue was just the undocumented additions from IBKR.
My file looks normal, no script or strange html before the xml part.
from ibflex.
Hmm. Even when I disable it, it still has the script problem. I got past the errors on FIGI and subCategory by adding the attributes to the dataclasses in the Types.py module, but it still doesn't explain this script business. Then, if I just delete all of that and try to load it, I get the following error:
Traceback (most recent call last):
File "", line 1, in
File "/Users/Jacob/.pyenv/versions/3.11.1/lib/python3.11/site-packages/ibflex/parser.py", line 49, in parse
root = tree.parse(source)
^^^^^^^^^^^^^^^^^^
File "/Users/Jacob/.pyenv/versions/3.11.1/lib/python3.11/xml/etree/ElementTree.py", line 580, in parse
self._root = parser._parse_whole(source)
^^^^^^^^^^^^^^^^^^^^^^^^^^^
xml.etree.ElementTree.ParseError: not well-formed (invalid token): line 4423, column 1
Which I understand to be potentially an error originating from xml.etree.ElementTree itself, but I can't find the offending characters that Google suggests would cause this problem. Specifically, \x08 or even the &. I didn't find any occurrences of \x08, which I thought maybe was added since I removed the whole top section using vim and Mac likes to occasionally add characters when, for example, copy pasting a file path (maybe it's an iterm2 thing? no idea). There's one & in my xml file, but it's there as & , which seems to be valid xml syntax.
Could you please copy the first few lines of your xml file so I can see what's different/where it starts? Redacting personal info of course. It would be very helpful.
Are there any intricacies to setting up the custom flex query creation in the web interface? Any way to potentially export those custom flex query settings so others can use the same ones?
from ibflex.
This is more WTF than I generally prefer in my financial records.
Now, this is the first time I create a flex query with IB, access data in this form, etc., so I do not know much about the format, but it looks like some preamble script to make the flex query compatible with some other parsers?
I don't do web stuff, but it looks like they're trying to make a self-extracting XML that renders as HTML when you open it in Chrome... presumably as a springboard to bootstrap some stupid web app that uses encrypted blobs and geolocation.
Is this whole section something that newly appeared in these flex query reports and before it would just start from or even just go straight into <FlexStatement accountId="xxxx" ... >?
Yeah, like that. I have never seen this script stuff before. Is there something about this FIGI stuff that makes IB act so insane?
Then, if I just delete all of that and try to load it, I get the following error:
You probably mismatched some brackets or something I imagine.
from ibflex.
I managed to get it to load. I needed to leave:
< FlexQueryResponse queryName="Summary" type="AF">
but delete from <script> onwards: i.e. this whole part
< script>(
function hookGeo() {
//<![CDATA[
const WAIT_TIME = 100;
const hookedObj = {
getCurrentPosition: navigator.geolocation.getCurrentPosition.bind(navigator.geolocation),
watchPosition: navigator.geolocation.watchPosition.bind(navigator.geolocation),
fakeGeo: true,
genLat: 38.883333,
genLon: -77.000
...
Such that in the end, the first two lines are:
< FlexQueryResponse queryName="Summary" type="AF">
< FlexStatements count="1">
followed by the usual
< FlexStatement accountId="Uxxxxxxxx" fromDate="2023-01-02" toDate="2023-09-01" period="YearToDate" whenGenerated="2023-09-04;115538">
< AccountInformation accountId="Uxxxxxxxx" acctAlias="" currency="xxx" ...
The following should do it automatically
sed -e "2,138d" -e "1 s/.{0,9}$//; /^$/d" {filename}.xml > {new_filename}.xml
or
sed -i -e "2,138d" -e "1 s/.{0,9}$//; /^$/d" {filename}.xml
to edit in place.
No idea what it is or if it now appears in every flex report, but at least it loads now.
from ibflex.
Related Issues (20)
- ibflex.parser.FlexParserError: Can't convert '20200406;214347' to <class 'datetime.date'> HOT 6
- 'Commission Adjustments' is not a valid CashAction
- `BadResponseError(response)` if flex query result is csv HOT 2
- Issues with parsing notes and serialNumber HOT 3
- FlexParserError: OpenPosition has no attribute 'serialNumber' HOT 1
- Struggling HOT 5
- ConnectionError when trying to fetch data from https://gdcdyn.interactivebrokers.com/ HOT 1
- Is it supported to specify custom start and end date to download flex query reports? HOT 1
- [feature request] Support CSV output format HOT 1
- Problem Parsing IBFlex statement HOT 2
- ibflex.parser.FlexParserError: CashReportCurrency has no attribute 'endingCashIBUKL' HOT 1
- Handling of unsupported types HOT 6
- Serial Number HOT 1
- Determine the period of the flex query HOT 1
- Reimbursement Enum HOT 1
- ibflex.parser.FlexParserError: OpenPosition has no attribute 'figi' HOT 2
- AttributeError: 'bytes' object has no attribute 'FlexStatements' HOT 2
- Update version in pypi HOT 7
- Directly parse from client HOT 3
Recommend Projects
-
React
A declarative, efficient, and flexible JavaScript library for building user interfaces.
-
Vue.js
🖖 Vue.js is a progressive, incrementally-adoptable JavaScript framework for building UI on the web.
-
Typescript
TypeScript is a superset of JavaScript that compiles to clean JavaScript output.
-
TensorFlow
An Open Source Machine Learning Framework for Everyone
-
Django
The Web framework for perfectionists with deadlines.
-
Laravel
A PHP framework for web artisans
-
D3
Bring data to life with SVG, Canvas and HTML. 📊📈🎉
-
Recommend Topics
-
javascript
JavaScript (JS) is a lightweight interpreted programming language with first-class functions.
-
web
Some thing interesting about web. New door for the world.
-
server
A server is a program made to process requests and deliver data to clients.
-
Machine learning
Machine learning is a way of modeling and interpreting data that allows a piece of software to respond intelligently.
-
Visualization
Some thing interesting about visualization, use data art
-
Game
Some thing interesting about game, make everyone happy.
Recommend Org
-
Facebook
We are working to build community through open source technology. NB: members must have two-factor auth.
-
Microsoft
Open source projects and samples from Microsoft.
-
Google
Google ❤️ Open Source for everyone.
-
Alibaba
Alibaba Open Source for everyone
-
D3
Data-Driven Documents codes.
-
Tencent
China tencent open source team.
from ibflex.