{
"cells": [
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"from matplotlib import pyplot as plt\n",
"import numpy as np\n",
"from scipy import signal as sig\n",
"import struct\n",
"import random\n",
"import ipywidgets\n",
"import itertools\n",
"\n",
"import colorednoise\n",
"\n",
"np.set_printoptions(linewidth=240)"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"%matplotlib widget"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"sampling_rate = 10 # sp/s"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"#colorednoise.powerlaw_psd_gaussian(1, int(1e4))"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"# From https://github.com/mubeta06/python/blob/master/signal_processing/sp/gold.py\n",
"preferred_pairs = {5:[[2],[1,2,3]], 6:[[5],[1,4,5]], 7:[[4],[4,5,6]],\n",
" 8:[[1,2,3,6,7],[1,2,7]], 9:[[5],[3,5,6]], \n",
" 10:[[2,5,9],[3,4,6,8,9]], 11:[[9],[3,6,9]]}\n",
"\n",
"def gen_gold(seq1, seq2):\n",
" print(seq1.shape, seq2.shape)\n",
" gold = [seq1, seq2]\n",
" for shift in range(len(seq1)):\n",
" gold.append(seq1 ^ np.roll(seq2, -shift))\n",
" return gold\n",
"\n",
"def gold(n):\n",
" n = int(n)\n",
" if not n in preferred_pairs:\n",
" raise KeyError('preferred pairs for %s bits unknown' % str(n))\n",
" t0, t1 = preferred_pairs[n]\n",
" (seq0, _st0), (seq1, _st1) = sig.max_len_seq(n, taps=t0), sig.max_len_seq(n, taps=t1)\n",
" return gen_gold(seq0, seq1)"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"fig, ax = plt.subplots()\n",
"ax.matshow(gold(5))"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"def modulate(data, nbits=5):\n",
" # 0, 1 -> -1, 1\n",
" mask = np.array(gold(nbits))*2 - 1\n",
" \n",
" sel = mask[data>>1]\n",
" data_lsb_centered = ((data&1)*2 - 1)\n",
"\n",
" return (np.multiply(sel, np.tile(data_lsb_centered, (2**nbits-1, 1)).T).flatten() + 1) // 2"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"data = np.array(list(range(16)))\n",
"\n",
"mask = np.array(gold(5))*2 - 1\n",
" \n",
"sel = mask[data>>1]\n",
"data_lsb_centered = ((data&1)*2 - 1)\n",
"mask.shape, data.shape, sel.shape\n",
"\n",
"#fig, ax = plt.subplots()\n",
"#ax.plot(\n",
"np.multiply(sel, np.tile(data_lsb_centered, (2**5-1, 1)).T).flatten()"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"def correlate(sequence, nbits=5, decimation=1, mask_filter=lambda x: x):\n",
" mask = np.tile(np.array(gold(nbits))[:,:,np.newaxis]*2 - 1, (1, 1, decimation)).reshape((2**nbits + 1, (2**nbits-1) * decimation))\n",
"\n",
" sequence -= np.mean(sequence)\n",
" \n",
" return np.array([np.correlate(sequence, row, mode='full') for row in mask])"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"nbits = 5\n",
"decimation = 10\n",
"\n",
"foo = np.repeat(modulate(np.array(list(range(4))), nbits).astype(float), decimation)\n",
"bar = np.repeat(modulate(np.array(list(range(4))), nbits) * 2.0 - 1, decimation) * 1e-3\n",
"print('shapes', foo.shape, bar.shape)\n",
"\n",
"mask = np.tile(np.array(gold(nbits))[:,:,np.newaxis]*2 - 1, (1, 1, decimation)).reshape((2**nbits + 1, (2**nbits-1) * decimation))\n",
"print('mask', mask.shape)\n",
"\n",
"fig, (ax1, ax2) = plt.subplots(2, figsize=(16, 5))\n",
"fig.tight_layout()\n",
"corr_m = np.array([np.correlate(foo, row, mode='full') for row in mask])\n",
"#corr_m = np.array([row for row in mask])\n",
"ax1.matshow(corr_m, aspect='auto')\n",
"#ax.matshow(foo.reshape(32, 31)[::2,:])\n",
"ax2.matshow(correlate(bar, decimation=decimation), aspect='auto')"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"decimation = 10\n",
"\n",
"fig, (ax1, ax2) = plt.subplots(2, figsize=(12, 5))\n",
"fig.tight_layout()\n",
"\n",
"#mask = np.tile(np.array(gold(nbits))[:,:,np.newaxis]*2 - 1, (1, 1, decimation)).reshape((2**nbits + 1, (2**nbits-1) * decimation))\n",
"#mask_stretched = np.tile(np.array(gold(nbits))[:,:,np.newaxis]*2 - 1, (1, 1, 1)).reshape((2**nbits + 1, (2**nbits-1) * 1))\n",
"\n",
"#ax1.matshow(mask)\n",
"#ax2.matshow(mask_stretched, aspect='auto')\n",
"\n",
"foo = np.repeat(modulate(np.array(list(range(4)))).astype(float), 1).reshape((4, 31))\n",
"foo_stretched = np.repeat(modulate(np.array(list(range(4)))).astype(float), 10).reshape(4, 310)\n",
"\n",
"ax1.matshow(foo)\n",
"ax2.matshow(foo_stretched, aspect='auto')"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"decimation = 10\n",
"signal_amplitude = 2.0\n",
"nbits = 5\n",
"\n",
"foo = np.repeat(modulate(np.array([0, 1, 0, 0, 1, 1, 1, 0]), nbits) * 2.0 - 1, decimation) * signal_amplitude\n",
"noise = colorednoise.powerlaw_psd_gaussian(1, len(foo))\n",
"\n",
"sosh = sig.butter(4, 0.01, btype='highpass', output='sos', fs=decimation)\n",
"sosl = sig.butter(6, 1.0, btype='lowpass', output='sos', fs=decimation)\n",
"filtered = sig.sosfilt(sosh, sig.sosfilt(sosl, foo + noise))\n",
"#filtered = sig.sosfilt(sosh, foo + noise)\n",
"\n",
"fig, ((ax1, ax3), (ax2, ax4)) = plt.subplots(2, 2, figsize=(16, 9))\n",
"fig.tight_layout()\n",
"\n",
"ax1.plot(foo + noise)\n",
"ax1.plot(foo)\n",
"ax1.set_title('raw')\n",
"\n",
"ax2.plot(filtered)\n",
"ax2.plot(foo)\n",
"ax2.set_title('filtered')\n",
"\n",
"ax3.plot(correlate(foo + noise, nbits=nbits, decimation=decimation))\n",
"ax3.set_title('corr raw')\n",
" \n",
"ax3.grid()\n",
"\n",
"ax4.plot(correlate(filtered, nbits=nbits, decimation=decimation))\n",
"ax4.set_title('corr filtered')\n",
"ax4.grid()\n",
"\n",
"rms = lambda x: np.sqrt(np.mean(np.square(x)))\n",
"rms(foo), rms(noise)"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"with open('data/raw_freq.bin', 'rb') as f:\n",
" mains_noise = np.copy(np.frombuffer(f.read(), dtype='float32'))\n",
" print('mean:', np.mean(mains_noise))\n",
" mains_noise -= np.mean(mains_noise)"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"decimation = 10\n",
"signal_amplitude = 2.0e-3\n",
"nbits = 6\n",
"\n",
"#test_data = np.random.randint(0, 2, 100)\n",
"#test_data = np.array([0, 1, 0, 0, 1, 1, 1, 0])\n",
"test_data = np.random.RandomState(seed=0xcbb3b8cf).randint(0, 2 * (2**nbits), 128)\n",
"#test_data = np.random.RandomState(seed=0).randint(0, 8, 64)\n",
"#test_data = np.array(list(range(8)) * 8)\n",
"#test_data = np.array([0, 1] * 32)\n",
"#test_data = np.array(list(range(64)))\n",
"\n",
"foo = np.repeat(modulate(test_data, nbits) * 2.0 - 1, decimation) * signal_amplitude\n",
"noise = np.resize(mains_noise, len(foo))\n",
"#noise = 0\n",
"\n",
"sosh = sig.butter(3, 0.01, btype='highpass', output='sos', fs=decimation)\n",
"sosl = sig.butter(3, 0.8, btype='lowpass', output='sos', fs=decimation)\n",
"#filtered = sig.sosfilt(sosh, sig.sosfilt(sosl, foo + noise))\n",
"filtered = sig.sosfilt(sosh, foo + noise)\n",
"\n",
"cor1 = correlate(foo + noise, nbits=nbits, decimation=decimation)\n",
"#cor2 = correlate(filtered, nbits=nbits, decimation=decimation)\n",
"\n",
"#cor2_pe = correlate(filtered, nbits=nbits, decimation=decimation, mask_filter=lambda mask: sig.sosfilt(sosh, sig.sosfiltfilt(sosl, mask)))\n",
"\n",
"sosn = sig.butter(12, 4, btype='highpass', output='sos', fs=decimation)\n",
"#cor1_flt = sig.sosfilt(sosn, cor1)\n",
"#cor2_flt = sig.sosfilt(sosn, cor2)\n",
"#cor1_flt = cor1[1:] - cor1[:-1]\n",
"#cor2_flt = cor2[1:] - cor2[:-1]\n",
"\n",
"fig, ((ax1, ax3), (ax2, ax4)) = plt.subplots(2, 2, figsize=(16, 9))\n",
"fig.tight_layout()\n",
"\n",
"ax1.plot(foo + noise)\n",
"ax1.plot(foo)\n",
"ax1.set_title('raw')\n",
"ax1.grid(axis='y')\n",
"\n",
"ax2.plot(filtered)\n",
"ax2.plot(foo)\n",
"ax2.set_title('filtered')\n",
"ax2.grid(axis='y')\n",
"\n",
"for i in range(0, len(foo) + 1, decimation*(2**nbits - 1)):\n",
" ax1.axvline(i, color='gray', alpha=0.5, lw=1)\n",
" ax2.axvline(i, color='gray', alpha=0.5, lw=1)\n",
"\n",
"for i, (color, trace) in enumerate(zip(plt.cm.winter(np.linspace(0, 1, cor1.shape[0])), cor1.T)):\n",
" if i%3 == 0:\n",
" ax3.plot(trace + 0.5 * i, alpha=1.0, color=color)\n",
"ax3.set_title('corr raw')\n",
"ax3.grid()\n",
"\n",
"#ax4.plot(cor2[:4].T)\n",
"#ax4.set_title('corr filtered')\n",
"#ax4.grid()\n",
"ax4.matshow(cor1, aspect='auto')\n",
"\n",
"#ax5.plot(cor1_flt)\n",
"#ax5.set_title('corr raw (highpass)')\n",
"#ax5.grid()\n",
"\n",
"#ax6.plot(cor2_flt)\n",
"#ax6.set_title('corr filtered (highpass)')\n",
"#ax6.grid()\n",
"\n",
"#ax6.plot(cor2_pe[:4].T)\n",
"#ax6.set_title('corr filtered w/ mask preemphasis')\n",
"#ax6.grid()\n",
"\n",
"rms = lambda x: np.sqrt(np.mean(np.square(x)))\n",
"rms(foo), rms(noise)"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"fig, ax = plt.subplots()\n",
"\n",
"seq = np.repeat(gold(6)[29]*2 -1, decimation)\n",
"sosh = sig.butter(3, 0.01, btype='highpass', output='sos', fs=decimation)\n",
"sosl = sig.butter(3, 0.8, btype='lowpass', output='sos', fs=decimation)\n",
"seq_filtered = sig.sosfilt(sosh, sig.sosfiltfilt(sosl, seq))\n",
"#seq_filtered = sig.sosfilt(sosh, seq)\n",
"\n",
"ax.plot(seq)\n",
"ax.plot(seq_filtered)"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"fig, axs = plt.subplots(3, 1, figsize=(9, 7), sharex=True)\n",
"fig.tight_layout()\n",
"axs = axs.flatten()\n",
"for ax in axs:\n",
" ax.grid()\n",
"\n",
"seq = np.repeat(gold(6)[29]*2 -1, decimation)\n",
"sosh = sig.butter(3, 0.1, btype='highpass', output='sos', fs=decimation)\n",
"sosl = sig.butter(3, 0.8, btype='lowpass', output='sos', fs=decimation)\n",
"cor2_pe_flt = sig.sosfilt(sosh, cor2_pe)\n",
"cor2_pe_flt2 = sig.sosfilt(sosh, sig.sosfiltfilt(sosl, cor2_pe))\n",
"\n",
"axs[0].plot(cor2_pe)\n",
"axs[1].plot(cor2_pe_flt)\n",
"axs[2].plot(cor2_pe_flt2)\n",
"\n",
"#for idx in np.where(np.abs(cor2_pe_flt2) > 0.5)\n"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"fig, ax = plt.subplots()\n",
"nonlinear_distance = lambda x: 100**(2*np.abs(0.5-x%1)) / (np.abs(x)+3)**2\n",
"x = np.linspace(-1.5, 5.5, 10000)\n",
"ax.plot(x, nonlinear_distance(x))"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"threshold_factor = 4.0\n",
"power_avg_width = 1024\n",
"max_lookahead = 6.5\n",
"\n",
"bit_period = (2**nbits) * decimation\n",
"peak_group_threshold = 0.1 * bit_period\n",
"\n",
"cor_an = cor1\n",
"\n",
"#fig, (ax1, ax2, ax3) = plt.subplots(3, figsize=(12, 12))\n",
"fig, (ax1, ax3) = plt.subplots(2, figsize=(12, 5))\n",
"fig.tight_layout()\n",
"\n",
"#ax1.matshow(sig.cwt(cor_an, sig.ricker, np.arange(1, 31)), aspect='auto')\n",
"\n",
"#for i in np.linspace(1, 10, 19):\n",
"# offx = 5*i\n",
"# ax2.plot(sig.cwt(cor_an, sig.ricker, [i]).flatten() + offx, color='red')\n",
"#\n",
"# ax2.text(-50, offx, f'{i:.1f}',\n",
"# horizontalalignment='right',\n",
"# verticalalignment='center',\n",
"# color='black')\n",
"#ax2.grid()\n",
"\n",
"ax3.grid()\n",
"print('cor_an', cor_an.shape)\n",
"\n",
"cwt_res = np.array([ sig.cwt(row, sig.ricker, [0.73 * decimation]).flatten() for row in cor_an ])\n",
"ax3.plot(cwt_res.T)\n",
"#def update(w = 1.0 * decimation):\n",
"# line.set_ydata(sig.cwt(cor_an, sig.ricker, [w]).flatten())\n",
"# fig.canvas.draw_idle()\n",
"#ipywidgets.interact(update)\n",
"\n",
"print('cwt_res', cwt_res.shape)\n",
"th = np.array([ np.convolve(np.abs(row), np.ones((power_avg_width,))/power_avg_width, mode='same') for row in cwt_res ])\n",
"ax1.plot(th.T)\n",
"print('th', th.shape)\n",
"\n",
"def compare_th(elem):\n",
" idx, (th, val) = elem\n",
" #print('compare_th:', th.shape, val.shape)\n",
" return np.any(np.abs(val) > th*threshold_factor)\n",
"\n",
"print([ (a.shape, b.shape) for a, b in zip(th.T, cwt_res.T) ][:5])\n",
"\n",
"peaks = [ list(group) for val, group in itertools.groupby(enumerate(zip(th.T, cwt_res.T)), compare_th) if val ]\n",
"print('peaks:', len(peaks))\n",
"peak_group = []\n",
"for group in peaks:\n",
" pos = np.mean([idx for idx, _val in group])\n",
" pol = np.mean([max(val.min(), val.max(), key=abs) for _idx, (_th, val) in group])\n",
" pol_idx = np.argmax(np.bincount([ np.argmax(np.abs(val)) for _idx, (_th, val) in group ]))\n",
" #print(f'group', pos, pol, pol_idx)\n",
" #for pol, (_idx, (_th, val)) in zip([max(val.min(), val.max(), key=abs) for _idx, (_th, val) in group], group):\n",
" # print(' ', pol, val)\n",
" ax3.axvline(pos, color='cyan', alpha=0.3)\n",
" \n",
" if not peak_group or pos - peak_group[-1][1] > peak_group_threshold:\n",
" if peak_group:\n",
" peak_pos = peak_group[-1][3]\n",
" ax3.axvline(peak_pos, color='red', alpha=0.6)\n",
" #ax3.text(peak_pos-20, 2.0, f'{0 if pol < 0 else 1}', horizontalalignment='right', verticalalignment='center', color='black')\n",
" \n",
" peak_group.append((pos, pos, pol, pos, pol_idx))\n",
" #ax3.axvline(pos, color='cyan', alpha=0.5)\n",
" \n",
" else:\n",
" group_start, last_pos, last_pol, peak_pos, last_pol_idx = peak_group[-1]\n",
" \n",
" if abs(pol) > abs(last_pol):\n",
" #ax3.axvline(pos, color='magenta', alpha=0.5)\n",
" peak_group[-1] = (group_start, pos, pol, pos, pol_idx)\n",
" else:\n",
" #ax3.axvline(pos, color='blue', alpha=0.5)\n",
" peak_group[-1] = (group_start, pos, last_pol, peak_pos, last_pol_idx)\n",
"\n",
"avg_peak = np.mean(np.abs(np.array([last_pol for _1, _2, last_pol, _3, _4 in peak_group])))\n",
"print('avg_peak', avg_peak)\n",
"\n",
"noprint = lambda *args, **kwargs: None\n",
"def mle_decode(peak_groups, print=print):\n",
" peak_groups = [ (pos, pol, idx) for _1, _2, pol, pos, idx in peak_groups ]\n",
" candidates = [ (0, [(pos, pol, idx)]) for pos, pol, idx in peak_groups if pos < bit_period*2.5 ]\n",
" \n",
" while candidates:\n",
" chain_candidates = []\n",
" for chain_score, chain in candidates:\n",
" pos, ampl, _idx = chain[-1]\n",
" score_fun = lambda pos, npos, npol: abs(npol)/avg_peak + nonlinear_distance((npos-pos)/bit_period)\n",
" next_candidates = sorted([ (score_fun(pos, npos, npol), npos, npol, nidx) for npos, npol, nidx in peak_groups if pos < npos < pos + bit_period*max_lookahead ], reverse=True)\n",
" \n",
" print(f' candidates for {pos}, {ampl}:')\n",
" for score, npos, npol, nidx in next_candidates:\n",
" print(f' {score:.4f} {npos:.2f} {npol:.2f} {nidx:.2f}')\n",
" \n",
" nch, cor_len = cor_an.shape\n",
" if cor_len - pos < 1.5*bit_period or not next_candidates:\n",
" score = sum(score_fun(opos, npos, npol) for (opos, _opol, _oidx), (npos, npol, _nidx) in zip(chain[:-1], chain[1:])) / len(chain)\n",
" yield score, chain\n",
" \n",
" else:\n",
" print('extending')\n",
" for score, npos, npol, nidx in next_candidates[:3]:\n",
" if score > 0.5:\n",
" new_chain_score = chain_score * 0.9 + score * 0.1\n",
" chain_candidates.append((new_chain_score, chain + [(npos, npol, nidx)]))\n",
" print('chain candidates:')\n",
" for score, chain in sorted(chain_candidates, reverse=True):\n",
" print(' ', [(score, [(f'{pos:.2f}', f'{pol:.2f}') for pos, pol, _idx in chain])])\n",
" candidates = [ (chain_score, chain) for chain_score, chain in sorted(chain_candidates, reverse=True)[:10] ]\n",
"\n",
"res = sorted(mle_decode(peak_group, print=noprint), reverse=True)\n",
"#for i, (score, chain) in enumerate(res):\n",
"# print(f'Chain {i}@{score:.4f}: {chain}')\n",
"(_score, chain), *_ = res\n",
"\n",
"def viz(chain):\n",
" last_pos = None\n",
" for pos, pol, nidx in chain:\n",
" if last_pos:\n",
" delta = int(round((pos - last_pos) / bit_period))\n",
" if delta > 1:\n",
" print(f'skipped {delta} symbols at {pos}')\n",
" for i in range(delta-1):\n",
" yield None\n",
" ax3.axvline(pos, color='blue', alpha=0.5)\n",
" decoded = nidx*2 + (0 if pol < 0 else 1)\n",
" yield decoded\n",
" ax3.text(pos-20, 0.0, f'{decoded}', horizontalalignment='right', verticalalignment='center', color='black')\n",
"\n",
" last_pos = pos\n",
"\n",
"decoded = list(viz(chain))\n",
"print('decoding [ref|dec]:')\n",
"failures = 0\n",
"for i, (ref, found) in enumerate(itertools.zip_longest(test_data, decoded)):\n",
" print(f'{ref or -1:>3d}|{found or -1:>3d} {\"✔\" if ref==found else \"✘\" if found else \" \"}', end=' ')\n",
" if ref != found:\n",
" failures += 1\n",
" if i%8 == 7:\n",
" print()\n",
"print(f'Symbol error rate e={failures/len(test_data)}')\n",
"print(f'maximum bitrate r={sampling_rate / decimation / (2**nbits) * nbits * (1 - failures/len(test_data)) * 3600} b/h')\n",
"#ax3.plot(th)"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"fig, axs = plt.subplots(2, 1, figsize=(9, 7))\n",
"fig.tight_layout()\n",
"axs = axs.flatten()\n",
"for ax in axs:\n",
" ax.grid()\n",
" \n",
"axs[0].plot(cor2_pe_flt2[1::10] - cor2_pe_flt2[:-1:10])\n",
"a, b = cor2_pe_flt2[1::10] - cor2_pe_flt2[:-1:10], np.array([0.0, -0.5, 1.0, -0.5, 0.0])\n",
"axs[1].plot(np.convolve(a, b, mode='full'))"
]
}
],
"metadata": {
"kernelspec": {
"display_name": "ma-thesis-env",
"language": "python",
"name": "ma-thesis-env"
},
"language_info": {
"codemirror_mode": {
"name": "ipython",
"version": 3
},
"file_extension": ".py",
"mimetype": "text/x-python",
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.9.2"
}
},
"nbformat": 4,
"nbformat_minor": 4
}