Flows stay in Pending_Remove for programmed subscriber on deleted ONU

Upon deletion of an ONU with a provisioned subscriber the ONU gets deleted correctly from VOLTHA and the ports get removed from ONOS but the flow remain in PENDING_REMOVE in ONOS because the flows are not cleared from the logical device agent list.

sdn@odtn:~/voltha/kind-voltha$ voltctl device delete $(voltctl device list --filter Type~brcm_openomci_onu -q)
ecc773a8-e853-4836-8052-c4035e2b4e62
sdn@odtn:~/voltha/kind-voltha$ voltctl device list
ID TYPE ROOT PARENTID SERIALNUMBER ADMINSTATE OPERSTATUS CONNECTSTATUS REASON
d17e6042-a2e1-4a4c-a534-6ca3aaf21ca5 openolt true 251f1ca2-aa50-4215-8f8d-6faeb717d8b9 BBSIM_OLT_0 ENABLED ACTIVE REACHABLE

onos@root > ports 10:55:26
id=of:00000a0a0a0a0a00, available=true, local-status=connected 16m55s ago, role=MASTER, type=SWITCH, mfr=VOLTHA Project, hw=open_pon, sw=open_pon, serial=BBSIM_OLT_0, chassis=a0a0a0a0a00, driver=voltha, channelId=10.244.1.4:51422, managementAddress=10.244.1.4, protocol=OF_13
port=1048576, state=enabled, type=fiber, speed=0 , adminState=enabled, portMac=0a:0a:0a:0a:0a:00, portName=nni-1048576

sdn@odtn:~/voltha/kind-voltha$ voltctl logicaldevice flows 251f1ca2-aa50-4215-8f8d-6faeb717d8b9
ID TABLEID PRIORITY COOKIE INPORT VLANID ETHTYPE IPPROTO UDPSRC UDPDST METADATA SETVLANID POPVLAN PUSHVLANID OUTPUT GOTOTABLE WRITEMETADATA METERID
6fed2c5c5b5738c8 0 10000 ~f81586a7 1048576 0x0800 17 67 68 CONTROLLER
4badc1ed5c90bee4 0 10000 ~ce6c3527 1048576 0x88cc CONTROLLER
8425983ca91d92a7 0 10000 ~ba31a4f2 16 900 0x0800 17 68 67 CONTROLLER 0x0000004000000000 1
9aa358748d836062 1 1000 ~4df91e40 1048576 900 0x0000000000000010 0 16 0x0000004000000000 1
3b736ff906b34422 0 1000 ~531d5ec9 1048576 900 0x0000000000000384 yes 1 0x0384004000000010 1
32993d1b5f993dd8 1 1000 ~986cca9a 16 900 900 0x8100 1048576 0x0000004000000000 1
b6c436df4222c730 0 1000 ~3fd5629a 16 0 900 1 0x0384004000100000 1
9c9b492daec3d7c6 0 10000 ~5eb48e6a 16 900 0x888e CONTROLLER 0x0384004000000000 1

onos@root > flows 10:54:54
deviceId=of:00000a0a0a0a0a00, flowRuleCount=8
id=c100005eb48e6a, state=PENDING_REMOVE, bytes=0, packets=0, duration=0, liveType=UNKNOWN, priority=10000, tableId=0, appId=org.opencord.olt, selector=[IN_PORT:16, ETH_TYPE:eapol, VLAN_VID:900], treatment=DefaultTrafficTreatment

{immediate=[OUTPUT:CONTROLLER], deferred=[], transition=None, meter=[METER:1], cleared=false, StatTrigger=null, metadata=METADATA:384004000000000/0}

id=c10000ba31a4f2, state=PENDING_REMOVE, bytes=0, packets=0, duration=0, liveType=UNKNOWN, priority=10000, tableId=0, appId=org.opencord.olt, selector=[IN_PORT:16, ETH_TYPE:ipv4, VLAN_VID:900, IP_PROTO:17, UDP_SRC:68, UDP_DST:67], treatment=DefaultTrafficTreatment

{immediate=[OUTPUT:CONTROLLER], deferred=[], transition=None, meter=[METER:1], cleared=false, StatTrigger=null, metadata=METADATA:4000000000/0}

id=c10000ce6c3527, state=ADDED, bytes=0, packets=0, duration=0, liveType=UNKNOWN, priority=10000, tableId=0, appId=org.opencord.olt, selector=[IN_PORT:1048576, ETH_TYPE:lldp], treatment=DefaultTrafficTreatment

{immediate=[OUTPUT:CONTROLLER], deferred=[], transition=None, meter=[], cleared=false, StatTrigger=null, metadata=null}

id=c10000f81586a7, state=ADDED, bytes=0, packets=0, duration=0, liveType=UNKNOWN, priority=10000, tableId=0, appId=org.opencord.olt, selector=[IN_PORT:1048576, ETH_TYPE:ipv4, IP_PROTO:17, UDP_SRC:67, UDP_DST:68], treatment=DefaultTrafficTreatment

{immediate=[OUTPUT:CONTROLLER], deferred=[], transition=None, meter=[], cleared=false, StatTrigger=null, metadata=null}

id=c100003fd5629a, state=PENDING_REMOVE, bytes=0, packets=0, duration=0, liveType=UNKNOWN, priority=1000, tableId=0, appId=org.opencord.olt, selector=[IN_PORT:16, VLAN_VID:0], treatment=DefaultTrafficTreatment

{immediate=[VLAN_ID:900], deferred=[], transition=TABLE:1, meter=[METER:1], cleared=false, StatTrigger=null, metadata=METADATA:384004000100000/0}

id=c10000531d5ec9, state=PENDING_REMOVE, bytes=0, packets=0, duration=0, liveType=UNKNOWN, priority=1000, tableId=0, appId=org.opencord.olt, selector=[IN_PORT:1048576, METADATA:384, VLAN_VID:900], treatment=DefaultTrafficTreatment

{immediate=[VLAN_POP], deferred=[], transition=TABLE:1, meter=[METER:1], cleared=false, StatTrigger=null, metadata=METADATA:384004000000010/0}

id=c100004df91e40, state=PENDING_REMOVE, bytes=0, packets=0, duration=0, liveType=UNKNOWN, priority=1000, tableId=1, appId=org.opencord.olt, selector=[IN_PORT:1048576, METADATA:10, VLAN_VID:900], treatment=DefaultTrafficTreatment

{immediate=[VLAN_ID:0, OUTPUT:16], deferred=[], transition=None, meter=[METER:1], cleared=false, StatTrigger=null, metadata=METADATA:4000000000/0}

id=c10000986cca9a, state=PENDING_REMOVE, bytes=0, packets=0, duration=0, liveType=UNKNOWN, priority=1000, tableId=1, appId=org.opencord.olt, selector=[IN_PORT:16, VLAN_VID:900], treatment=DefaultTrafficTreatment

{immediate=[VLAN_PUSH:vlan, VLAN_ID:900, OUTPUT:1048576], deferred=[], transition=None, meter=[METER:1], cleared=false, StatTrigger=null, metadata=METADATA:4000000000/0}