20170702

Javascript to filter and count records

Github was talking about open source Fridays ... what's wrong with open source weekemds?

I needed something like this but couldn't find it on the web.... so time to write it myself. The idea is to have a python routine compile a set of data (which I can't share so it just generates some date related data) and then use a webpage with javascript to present record counts for different fields.

The top line of buttons lets you select which field to show, by default it shows all fields.
The second line shows the filters that are currently active, click on one of them to remove it again or 'Clear All' to clear them all in one go.

Then there is a table with the results, click on any of the field values to create a filter for that field and value combination. Currently there is no way to have filters for multiple values of the same field.

Lessons learned:

  • mixing javascript and html: it took me a while to figure out that when you pass an array through javascript to a html button,
  • the inspect feature makes life so much easier...,
  • in order to copy an array you can't just assign it, you need to use .slice() to make a deep copy.
Answers found on stackoverflow:



Try it out: count and filter test.

And here is the code:

<html>
<head><title>count and filter test</title>
</head>
<body onload="generate_data();filterandcount('all',[],[]);"> 

<p id="controls"> </p>
<p id="filters"> </p>
<p id="result_table"> </p>

<script>
// additional features that could be implemented:
// 1. show subset/all info for the selected filter, count becomes a button to show this


function onlyUnique(value, index, self) 
{ 
 return self.indexOf(value) === index;
}

var records=[];
var filter_field_array=[];
var filter_value_array=[];


function generate_data()
{
 // generate some data
 //var records=[];

 var base=[2014,1,1,0,0,0];
 var range=[4,12,31,23,59,59];
 var len=[4,2,2,2,2,2];

 var num_records=1000;
 for (var i=0; i<6; i++) 
 {
  var t=[];
  for (var j=0; j<num_records; j++) 
  { 
   var t2="0000"+(Math.floor(Math.random()*range[i])+base[i]);
   t2=t2.substring(t2.length-len[i]);
   t.push(t2);
  }
  records.push(t);
 }

 for (var i=0; i<5; i++)
 {
  var t=[];
  for (var j=0; j<num_records; j++) 
  { 
   var t2=records[0][j];
   for (var k=0; k<(i+1); k++)
   {
    t2+=records[k+1][j];
   }
   t.push(t2);
  }
  records.push(t);
 }
}

function filterandcount(do_field,filter_field_array_p,filter_value_array_p) 
{
 //console.log(do_field);

// console.log(typeof 

 var filter_field_array=[];
 var filter_value_array=[];

 if (filter_field_array_p instanceof Array) 
 {
  filter_field_array=filter_field_array_p.slice(); 
 }
 else
 {
  filter_field_array.push(filter_field_array_p);
 }

 if (filter_value_array_p instanceof Array)
 {
  filter_value_array=filter_value_array_p.slice();
 }
 else
 {
  filter_value_array.push(filter_value_array_p);
 }



 var field_names = ['year','month','date','hour','minute','second','y+month','y+m+day','y+m+d+hour','y-m-d-h-minute','y-m-d-h-m-second'];


 var controls_string="Count: <button type='button' onclick=\"filterandcount('all',["+filter_field_array+"],["+filter_value_array+"])\">All</button>";
 for (var i=0; i<field_names.length; i++) 
 {
  controls_string+="<button type='button' onclick=\"filterandcount('"+field_names[i]+"',["+filter_field_array+"],["+filter_value_array+"])\">"+field_names[i]+"</button>";
 }
 document.getElementById("controls").innerHTML=controls_string;

 var t_filter_value="";
 if (filter_value_array.length==0)
 {
  t_filter_value="&lt;none&gt;";
 }
 else
 {
  // remove duplicate values - the user should not be doing this!
  var t_filter_field_array=[];
  var t_filter_value_array=[];
  for (var i=0; i<filter_value_array.length; i++)
  {
   // only add them if we can't find them in the temp arrays
   var t=0;
   for (var j=0; j<t_filter_value_array.length; j++)
   {
    if (filter_field_array[i]==t_filter_field_array[j])
    {
     if (filter_value_array[i]==t_filter_value_array[j])
     {
      t+=1;
     }
    }
   }
   if (t==0)
   {
    t_filter_field_array.push(filter_field_array[i]);
    t_filter_value_array.push(filter_value_array[i]);
   }
  }
  filter_field_array=t_filter_field_array.slice();
  filter_value_array=t_filter_value_array.slice();

  console.log(filter_field_array);
  console.log(filter_value_array);

  for (var i=0; i<filter_value_array.length; i++)
  {
   t_filter_field_array=[];
   t_filter_value_array=[];   
   // remove current values from arrays
   for (var j=0; j<filter_value_array.length; j++)
   {
    if (i!=j)
    {
     t_filter_field_array.push(filter_field_array[j]);
     t_filter_value_array.push(filter_value_array[j]);
    }
   }
   t_filter_value+="<button type='button' onclick=\"filterandcount('"+do_field+"',["+t_filter_field_array+"],["+t_filter_value_array+"])\">";
   t_filter_value+="<b>&#X2716</b> "+filter_value_array[i]+" on "+field_names[filter_field_array[i]]+"</button>";
  }
 }

 var filters_string="Filter: " +t_filter_value+"&nbsp;<button type='button' onclick=\"filterandcount('"+do_field+"',[],[])\">Clear All</button>";
 document.getElementById("filters").innerHTML=filters_string;

 // but we actually need to transpose the array so we can grab the unique values per array
 //transpose = m => m[0].map((x,i) => m.map(x => x[i]))
 //records2=transpose(records);
 // uhm, we're generating the data ourselves a couple lines up - so why transpose!
 // fixed!

 // instead of just assigned it to records2 run any filters first
 var records2=[];
 if (filter_field_array.length!=0)
 {
  // create a set of empty arrays
  for (var i=0; i<records.length; i++)
  {
   records2.push([]);
  }
  for (var i=0; i<records[0].length; i++)
  {
   var t=0;
   for (var j=0; j<filter_field_array.length; j++)
   {
    if (records[filter_field_array[j]][i]==filter_value_array[j])
    {
     t+=1;
    }
   }
   if (t==filter_field_array.length)
   {
    for (var j=0; j<records.length; j++)
    {
     records2[j].push(records[j][i]);
    }
   }
  }
 }
 else
 {
  records2=records;
 }
 //records2=records;

 // now get the unique values per sub-array
 var unique_records2=[]; //initialize empty array
 for (var i=0; i<records2.length; i++) 
 {
  // not optimized: only do the relevant sub-array(s)
  //if ((field_names[i]==do_field)||(do_field=="all"))
  //{
  unique_records2.push(records2[i].filter(onlyUnique));
  //}
  //else
  //{
  //unique_records2.push([]);
  //}
 }

 // now we need to count the values for those 
 // first we populate the count array with 0's
 var unique_count2=[];
 for (var i=0; i<unique_records2.length; i++) 
 {
  unique_count2.push([]);
  for (var j=0; j<unique_records2[i].length; j++) 
  {
   unique_count2[i].push(0);
  }
 }

 // now we can just increase whenever we have a match
 for (var i=0; i<records2.length; i++) 
 {
  for (var j=0; j<records2[i].length; j++) 
  {
   for (var k=0; k<unique_records2[i].length; k++) 
   {
    if (records2[i][j]==unique_records2[i][k])
    {
     unique_count2[i][k]+=1;
    }
   }
  }
 }

 // sort each fields value in descending order
 // use an array with pointers so we don't end up copying lots of data
 var unique_sort_order=[];
 for (var i=0; i<unique_count2.length; i++)
 {
  var t=[];
  // add an index to the array
  for (var j=0; j<unique_count2[i].length; j++) 
  {
   t[j]=[unique_count2[i][j],j];
  }
  t.sort(function(left, right) { return left[0] < right[0] ? -1 : 1; });
  t.reverse();
  var t2=[];
  for (var j=0; j<t.length; j++) 
  {
   t2.push(t[j][1]);
   //t2[j]=t[j][0];
  }
  unique_sort_order.push(t2);
 }

 // now replace the table in div with the contents
 table_string="<table border=1>";

 for (var i=0; i<field_names.length; i++) 
 {
  if ((field_names[i]==do_field)||(do_field=="all"))
  {
   var ti=filter_field_array.slice();
   ti.push(i);
   table_string +="<tr><td><b>"+field_names[i]+"</b></td></tr>";
   for (var j=0; j<unique_records2[i].length; j++) 
   {
    var ti2=filter_value_array.slice();
    ti2.push(unique_records2[i][unique_sort_order[i][j]]);

    //console.log(ti);
    //console.log(ti2);

    table_string +="<tr><td><button type='button' onclick=\"filterandcount('all',["+ti+"],["+ti2+"])\">"+unique_records2[i][unique_sort_order[i][j]]+"</button>";
    table_string +="</td><td>"+unique_count2[i][unique_sort_order[i][j]]+"</td></tr>";
   }
  }
 }
 table_string +="</table>";

 // simplified: print the array count values
 document.getElementById("result_table").innerHTML = table_string;
}
</script>

</body>
</html>

20160709

Painting the garage door

Monday:
too yellow...

Tuesday:
too red... 

Wednesday:
too purple...

Thursday:
too blue ...

Friday:
too green ....

Saturday:

just right! And that's enough painting for this year.



20160305

Linux on the Ockel Sirius B - part 2

It works!

Use the EUFI update that is included with the Ubuntu instructions on http://www.ockelproducts.com/support/#a6857fbe0003d9f3e to convert to a 64 bit UEFI. Now you can install a normal distro. Running Fedora Core 23, to get wireless working i did a modprobe on brcmfmac. I also copied the firmware from the Ockel website, not sure if that is required.

20160304

Linux on the Ockel Sirius B - part 1

Initially i thought this was going to be easy, unfortunately all distros i tried to boot from gave me exactly the same result: EUFI 32bit shell. Never ever booted from the usb stick.

Finally found these two posts:

Copied Debian Multiarch 8 on a usb stick using dd (from http://cdimage.debian.org/debian-cd/current/multi-arch/iso-dvd/ )

Switch off secure boot.

So now for the first time i got past the eufi shell!

Install runs ok, didn't find a network device (should be wireless), boots but doesn't go into graphics. Log in and startx doesn't seem to make it very happy: looks like the graphics mode is on,but still shows the prompt. Not that it does anything. I can switch to other terminal using ctrl+alt+Fx.


Update: saw that there is info on the Ockel website on how to do this: http://www.ockelproducts.com/support/#a6857fbe0003d9f3e will give that a shot later!

20140719

Riding to work with the Rubbee

First i did a test run last Sunday to the office (approx 15 miles) and after a short break back again:
  • to work 70 minutes
  • back home 70 minutes
  • only put the Rubbee on the wheel for the uphill sections, so had still a little bit spare when i got home
  • very nice ride: cloudy with 2 minutes of drizzle when i started out, sunny by the time i got to the office. Could have done with more water. No backpack.
  • on wet sections the noise the Rubbee made was quite bad ... slipping over the knobbly bits of the tire i guess.
Wednesday was the first for real:
  • to work 66 minutes, ran out of power just before the office (200m)
  • back home 64 minutes, ran out of power about a mile from home
  • using the Rubbee almost 75% of the time, only taking it off the wheel for the downhill sections
  • couldn't get my rhythm until i was almost at the office, just not used to exercise that early in the morning i guess...
  • cooler than Sumday on the way back
  • my front light warns that it's getting low on power after about 40 minutes by switching the indicator from green to orange, it is however still bright at the end of ride
  • also changed from knobbly tires to road tires (for a second i was tempted to go for near slicks ... but the roads are pretty bad so i went for a bit more sensible ones.
  • first time with a backpack, may also have contributed to the not so pleasant ride in, i haven't ridden with a backpack for ages.
Friday:
  • to work 68 minutes
  • back home 67 minutes
  • much better rider in, despite the roads being wet and me getting rather muddy, 
  • the way back was harder as it was hot! The backpack did not help...
  • drawback in the wet was that the Rubbee started slipping when i put on more power (seemed ok to keep the same speed).
  • forgot the recharge the front light, got home and it was still orange (and quite bright)

The Rubbee starts to run with much less resistance than when it was new, hope this continues to improve as it slows you down quite a bit. that's the reason for taking it off the wheel on the downhill sections, the cost is about two minutes each time and you have to slowback, wait for a gap between cars before you can accelerate.

Number of stops:
  • five on the way to work (for Wednesday and Friday)
  • four on the way back (again Wednesday and Friday only)
So there is the potential to shave off ten to twelve minutes on the way to work and eight to 10 on the way back, but only if the Rubbee runs near-frictionless (assumption is that there is a 30 second loss on slowing down and accelerating).
55 minutes each way sounds tempting... can the Rubbee do it or do i need a real electric mountain bike. I'd need to leave the car and the the ebike a thousand times to break even ... need to find a better reason than just financial i guess.

20140506

Never lose the locking pin on my Rubbee again!

On a long down hill section (public road in Surrey, rather bumpy due to the potholes) i had my Rubbee off the back wheel. And at the bottom it was on my wheel ... somewhere along the way i'd lost the locking pin (and i wasn't in the mood to cycle back up).

Never again! Here is my solution (ignore the yellow bits, that clip is just temporary until i find the one i want to use again... same for the temporary 'locking pin', i'm not going to ride with that!):



Clip it through the ring of the locking pin, so that if it falls out again i can just pop it back in.

20140106

Accessing IR-Blue-DM from linux

Note: do the initial tests with the bluetooth dongle straight on the machine (Udoo in my case) instead of plugged in a hub. Scanning works on a hub but for some reason rfcomm doesn't connect very often if the dongle is on a hub.

Quick hack #1:

Check if we can see the device:
root@imx6-qsdl:/home/ubuntu# hcitool scanScanning ...
    EC:FE:xx:xx:xx:xx    IR-Blue-DM


Same using LE:
root@imx6-qsdl:/home/ubuntu# hcitool lescan
LE Scan ...
EC:FE:xx:xx:xx:xx






Check if we need a pin:
root@imx6-qsdl:/home/ubuntu# bluez-simple-agent hci0 EC:FE:xx:xx:xx:xx
Release
New device (/org/bluez/5683/hci0/dev_EC_FE_xx_xx_xx_xx)

Seems we don't need one, connect using rfcomm.
root@imx6-qsdl:/home/ubuntu# rfcomm connect hci0 EC:FE:xx:xx:xx:xx        
Connected /dev/rfcomm0 to EC:FE:xx:xx:xx:xx on channel 1
Press CTRL-C for hangup
^CDisconnected






See if there is any data:
root@imx6-qsdl:/home/ubuntu# cat /dev/rfcomm0

And after you do 'echo >/dev/rfcomm0 "R001\r\n"' in another terminal there is data!:
E211208208208215214212212218220218213222223221218223223221216224225223220227226225222226227225223228228227225229229228225229228229228229230230228230230229230230230231230230230229229227229227227188188188188205205196196205205205196205205205205213205205205205213205205213213213205213213213205213213222213222222222213222222222222222222222222222222222222222222222222222222222222213222222222  0 17 17 10 41 62 62 52 79 86 96 90103121124107131152159138159162169148169183183155179186186162179193193159176190190165176186183148172172176131151158158120124138145 96103107110 65 69 75 68 23255255255255255255255255255255  0  0  0  0  0  0  0  0  0216221205  0  0  0  8150 26 17 93105 36230 79 40 34  0128  0  0255 33  0188 26190 59 25245 27150 26255 30116 91166 32136 44 10 33  1 26EX

R144006836145-00045I   -54   -54   -52   -54   -53   -54   -51   -50   -52   -50   -52   -49   -50   -50   -49   -47   -51   -51   -51   -49   -52   -50   -50   -48   -48   -50   -45   -47   -45   -43   -42   -44   -41   -42   -43   -45   -27   -41   -43   -44   -40   -41   -41   -43   -42   -42   -42   -42   -44   -43   -43   -42   -42   -43   -42   -42   -42   -41   -42   -40   -42   -42   -42   -41X



Quick hack #2:

Use a small python routine to read data:
import serial
import time

ser=serial.Serial("/dev/rfcomm0", baudrate=57600,timeout=0.1)

ser.open()
ser.write("R001"+chr(13)+chr(10))

while (1==1):

 d=ser.readline()
 print len(d),d


Success! Now to write something that gets temperature values and shows the result graphically...

I found the R001 command in the android code on http://github.com/RHWorkshop , the H command can be used to increase the speed at which the data is sent.

20130824

Teamviewer - using aliases

I really like teamviewer (free for personal use), after creating an account and running it on most of my machines and my android tablet the one thing i couldn't find easily figure out was why one of my machines had a human readable name and all the others showed up with just the teamviewer id.

The annoying thing is that only the machine with the human readable name has an option to change its properties and the others only allow me to connect to them. Googled for an answer for ages, must be a typical case of me searching using the wrong keywords or nobody else trying to do this...

Finally after messing around for ages i found that i had to add the machines manually in computers and contacts (not logical as they're in my list already...), then you get the option to add an alias (grumble: why do have to type in the teamviewer id - would like to see a rightclick option for that).

Great now i'll be able to connect to the correct machine in one go without having to memorize the teamview id.

As a 'benefit' the machines are now listed twice: directly in the 'my computer' list and once under my account. And there must be some logic as to why the machines show up yellow (idle) under the account but green (active) in the 'my computer' list.

20130703

Key-fob is sorted, can we do the same for URUK WiFi Router Module (Arduino Compatible)

Cool! The Pi-Lite key-fob (see yesterday's post:Get a Pi Lite and we'll all a Pi Lite key-fob as a bonus!) should be happening as they went over £10K today.

Can we do the same for URUK WiFi Router Module (Arduino Compatible) , i'd like to get one of those ... currently at just over $6K, there is another week to get it funded at $10K.

20130702

Get a Pi Lite and we'll all a Pi Lite key-fob as a bonus!

There is always something interesting on Kickstarter...

Pi Lite white - Bright white LED display for Raspberry Pi

If they reach £10K (easy, the project is over £9 already) we'll all get the Pi Lite acrylic key-fob as a bonus, so go on get one!

20130426

Don't have a 3D printer yet? Do something about it!

A couple of Kickstarters worth looking at:


Rigidbot (build volume: 10"x10"x10" /  25.4x25.4x25.4cm or 12"x16"x10" / 30x40x25.4cm)
Some people are worried that they're not going to reach all their stretch goals... The initial goal was only $31.5k, now to get the heated print bed they need 31.7x that, currently they at almost 60%. You can make it happen!


Cyrus (build volume: 10"x11"x12")
Almost half way to the goal, looks great. Like shiny things? Like Swarovski gems? Then go for the Great Reward whilst it's still available. Don't forget to add the heated bed, lcd and dual extruder!


So you choice is simple: get a Rigidbot. Or a Cyrus. Or a couple of both. Or 31 Cyruses / 750 t-shirts! Come on, what are you waiting for?

36
And whilst you're at it don't forget to get a couple of these: Doodle 3D Wireless 3d printing and a way to get the kids hooked also (kids: sell it to mom and dad as an educational tool and you'll get to play with the 3d printer once they're bored with it), what more can you ask for?

20130111

Arduino + Canbus shield + GPS Receiver - update 1

A quick update on my Arduino + Canbus shield + GPS Receiver post from last year.

I had this plan to put the files in a sub folder in order to make sure that i would hit the limit to the number of files in the root of the file system, added a bit of code that would create up to a thousand files. Somehow it didn't work: didn't want to write in the sub folder...

But why use separate files anyway? The files will be evaluated on my main machine, it's easy enough for it to split the file. That simplifies the code a lot:


void start_new_file(void) 
{
 digitalWrite(LED2, HIGH);

 // we want to add to the end of the file file and flush manually for performance and to be nice to the sd card
 file = SD.open("DATA.TXT", O_READ | O_APPEND | O_WRITE);
 digitalWrite(LED2, LOW);
}

Also spotted the post Why is the SD library slow? and decided to flush the file manually. Performance isn't really an issue but fewer writes to the SD won't hurt! I flush before checking for the click.

20121029

Android development

Ages ago i had a look at Google's development environment, found it rather complex and then it was discontinued.

Today i stumbled over some documentation based on http://www3.ntu.edu.sg/home/ehchua/programming/android/Android_HowTo.html which again was more complicated than it need to be.


My steps:

  1. install JDK from http://www.oracle.com/technetwork/java/javase/downloads/jdk7u9-downloads-1859576.html
  2. install Eclipse Classic 4.2.1 from http://www.eclipse.org/downloads/ and untar
  3. run eclipse and from Help -> Install New Software -> Add add https://dl-ssl.google.com/android/eclipse/
  4. restart eclipse when it asks for it
  5. from Windows -> Android SDK Manager install the tools, the android versions you're interested in etc
  6. now you can create a virtual android device from Windows -> AVD Manager.

20120929

64 core supercomputer for $199

Four weeks to go to raise $750k (over $80k already in two days). Get yours in May 2013 for $99:
http://www.kickstarter.com/projects/adapteva/parallella-a-supercomputer-for-everyone

20121003: Ah, not 64 core is it said in the title, 'only' 16. Still worthwhile! Or get two for $199 ... but how much is a 64core version going to be? Over $150k in six days, still on track to achieve the goal.

20121004: Ok, back to 64 core in the title now there is an option to get 64cores! Not yet on track to achieve that one - not even on track for the $750k ... looks like it'll be $35k too low.

20121027: Wow, funding was reached. Didn't think that was going to happy, funding currently stands at $780K and still half a day to go ... not likely that we'll reach $3m (to get 64 cores) but then i didn't think we'd reach $750k. Who knows, may still happen!

20120719

Raspberry Pi - Raspbian, Wheezy & sound

first SD card image based on the Raspbian distribution

Good:

  • based on Wheezy,
  • ssh enabled by default,
  • raspi-config makes it easy to extend the image to the entire sd card, configure how much memory goes to graphics etc.,
  • sound driver is loaded automatically,
Bad:
  • can't get audio to work from python-pyao, doesn't want to open any device i tried, even installed pulse (but audio isn't broken as mpg123 works fine...).

20120709

Arduino + Canbus shield + GPS Receiver


Ingredients:
Also required:

  • soldering iron
  • patience & basic soldering skills (to solder the headers onto the Canbus shield)
  • arduino software
  • computer, car/van etc.

Things to keep in mind:

  • when you want to stop logging: press on the joystick until the leds go out so that the file on the sd card is properly closed
  • make sure to reset if you plugin before starting the engine, not all pids are available at that point
  • with the engine off my engine runs at 280 rpm...
  • powered from the canbus, so i disconnect before leaving the vehicle in case it would drain the battery


Still todo:

  • solder on the lcd display, build an interface that shows fuel consumption
  • see if there is a more efficient way to log all available sensors (on startup  check which pids are available and then log all of the available ones)
  • print a case
  • sleep mode: if the engine is off (or rpm is 280 for a couple of times in a row), switch all leds off, switch off the gps (if possible), check every n seconds if the engine was started etc.


20120624

Raspberry Pi - alsa

Install alsa:

  • sudo apt-get alsa

and make sure that the module is loaded:

  •  sudo modprobe snd-bcm2835

20120623

Raspberry Pi - ssh


Using the Debian squeeze image from http://www.raspberrypi.org/downloads

I'd like to be able to run it headless, need ssh running automatically after boot:
  1. ssh-keygen (answer the prompts)
  2. sudo update-rc.d ssh default
  3. sudo reboot
or

  1. cd /boot
  2. sudo mv boot_enable_ssh.rc boot.rx
  3. sudo reboot



20120617

Robot hand from seven parts - 2 (some photos)












The finger bend both ways: the finger tips can therefore are either pads or sharp.

From the base of the wrist to the tip of a finger is about 27cm. If i ever get to build the entire robot it'll be approx 2m70 high...

Robot hand from seven parts - 1

Take one or more of each of the following parts, snap or friction fit them together and you get a robot hand!

Snap the first two together to create a rotating joint:


Hold a couple of those together with a connection piece and you get finger:
At one end of the finger place a finger tip:
Give the thumb an additional degree of rotation:

 Use a spacer between the fingers...
when you slide them on the wrist, then snap the thumb on and the hand is ready.

Partslists:

Part 1 Part 2Part 3Part 4
Part 5Part 6Part 7




Finger:
  • Part 1 : 3
  • Part 2 : 3
  • Part 3 : 3
  • Part 4 : 1

Thumb:

  • Part 1 : 4
  • Part 2 : 4
  • Part 3 : 4
  • Part 4 : 1
  • Part 5 :1

Hand:

  • Part 1 : 16
  • Part 2 : 16
  • Part 3 : 16
  • Part 4 : 5
  • Part 5 : 3
  • Part 6 : 4
  • Part 7 : 1