[Kea-users] Memory leak on dhcp6

classic Classic list List threaded Threaded
8 messages Options
Reply | Threaded
Open this post in threaded view
|

[Kea-users] Memory leak on dhcp6

Nitzan Tzelniker
Hi,

I am running two dhcpv6 server with high-availability
We have about 2500 regular leases and 2500 PD leases
The setup is working but it look like kea-dhcp6 leak memory 
When it start (with all of the leases in the file ) it take less then 1% of the memory but after two hours it take 28% and if I will give it 8 hours it will crash 

I don't see issue on the standby server only on the primary server

Anybody saw this behavior  
Any idea how to debug it

 kea-dhcp6 -V
1.5.0
tarball
linked with:
log4cplus 1.1.3
OpenSSL 1.0.2k-fips  26 Jan 2017
database:
Memfile backend 2.1

Running on CentOS 7.6.1810 VM kernel 3.10.0-957.1.3.el7.x86 with 2 * vCPU and 2GB RAM 

bellow is the dhcp6 config file omitting 102 subsets 

{
"Dhcp6": {
    "interfaces-config": {
        "interfaces": [ "ens192/1111:aef:aa77:83::100" ]
    },
    "mac-sources": [ "docsis-cmts", "client-link-addr-option" ],

    "control-socket": {
        "socket-type": "unix",
        "socket-name": "/tmp/kea-dhcp6-ctrl.sock"
    },

    "lease-database": {
        "type": "memfile",
        "lfc-interval": 3600
    },
    "expired-leases-processing": {
        "reclaim-timer-wait-time": 10,
        "flush-reclaimed-timer-wait-time": 25,
        "hold-reclaimed-time": 3600,
        "max-reclaim-leases": 100,
        "max-reclaim-time": 250,
        "unwarned-reclaim-cycles": 5
    },

    "renew-timer": 1000,
    "rebind-timer": 2000,
    "preferred-lifetime": 3000,
    "valid-lifetime": 4000,
    "option-data": [
        {
            "name": "dns-servers",
            "data": "1111:aef:aa77:1::22, 1111:aef:aa77:2::22"
        }
    ],
    "subnet6": [
        {
            "subnet": "1111:aec:0:10::/64",
            "pools": [ { "pool": "1111:aec:0:10::10-1111:aec:0:10:ffff::" } ],
            "pd-pools": [
                {
                    "prefix": "1111:aec:10::",
                    "prefix-len": 44,
                    "delegated-len": 56
                }
            ]
        },
//....
//  101 more subnets 
//....
 "hooks-libraries": [
        {
            "library": "/usr/local/lib/hooks/libdhcp_ha.so",
            "parameters": {
                "high-availability": [ {
                    "this-server-name": "KEA_DHCPv6_1",
                    "mode": "hot-standby",
                    "heartbeat-delay": 10000,
                    "max-response-delay": 20000,
                    "max-ack-delay": 10000,
                    "max-unacked-clients": 0,
                    "peers": [
                        {
                            "name": "KEA_DHCPv6_1",
                            "role": "primary",
                            "auto-failover": true
                        },
                        {
                            "name": "KEA_DHCPv6_2",
                            "role": "standby",
                            "auto-failover": true
                        }
                    ]
                } ]
            }
        }
    ]
},

"Logging":
{
  "loggers": [
    {
        "name": "kea-dhcp6",
        "output_options": [
            {
                "output": "/usr/local/var/log/kea-dhcp6.log"
            }
        ],
        "severity": "INFO",

        "debuglevel": 0
    }
  ]
}
}
Thanks

Nitzan

_______________________________________________
Kea-users mailing list
[hidden email]
https://lists.isc.org/mailman/listinfo/kea-users
Reply | Threaded
Open this post in threaded view
|

[Kea-users] Fwd: Memory leak on dhcp6

Nitzan Tzelniker
Hi,

It seems that the issue is related to the HA
After I commented the HA part the memory consumption is stayed low 
I am adding the relevant configuration if you have any idea

Thanks

Nitzan

Standby server ha configuration part: 

    "hooks-libraries": [
        {
            "library": "/usr/local/lib/hooks/libdhcp_lease_cmds.so",
            "parameters": { }
        },
        {
            "library": "/usr/local/lib/hooks/libdhcp_ha.so",
            "parameters": {
                "high-availability": [ {
                    "this-server-name": "KEA_DHCPv6_2",
                    "mode": "hot-standby",
                    "heartbeat-delay": 10000,
                    "max-response-delay": 20000,
                    "max-ack-delay": 10000,
                    "max-unacked-clients": 0,
                    "peers": [
                        {
                            "name": "KEA_DHCPv6_1",
                            "role": "primary",
                            "auto-failover": true
                        },
                        {
                            "name": "KEA_DHCPv6_2",
                            "role": "standby",
                            "auto-failover": true
                        }
                    ]
                } ]
            }
        }
    ]
}

kea-ctrl-agent.conf  from both servers (Only the ip address is different ) 

{

"Control-agent": {
    "http-host": "1.1.1.X",
    "http-port": 8080,

    "control-sockets": {
        "dhcp4": {
            "socket-type": "unix",
            "socket-name": "/tmp/kea-dhcp4-ctrl.sock"
        },
        "dhcp6": {
            "socket-type": "unix",
            "socket-name": "/tmp/kea-dhcp6-ctrl.sock"
        }
    },

    "hooks-libraries": [
    ]
},

"Logging":
{
  "loggers": [
    {
        "name": "kea-ctrl-agent",
        "output_options": [
            {
                "output": "/usr/local/var/log/kea-ctrl-agent.log"
            }
        ],
        "severity": "INFO",

        "debuglevel": 0
    }
  ]
}
}
---------- Forwarded message ---------
From: Nitzan Tzelniker <[hidden email]>
Date: Fri, May 3, 2019 at 10:13 PM
Subject: Memory leak on dhcp6
To: <[hidden email]>


Hi,

I am running two dhcpv6 server with high-availability
We have about 2500 regular leases and 2500 PD leases
The setup is working but it look like kea-dhcp6 leak memory 
When it start (with all of the leases in the file ) it take less then 1% of the memory but after two hours it take 28% and if I will give it 8 hours it will crash 

I don't see issue on the standby server only on the primary server

Anybody saw this behavior  
Any idea how to debug it

 kea-dhcp6 -V
1.5.0
tarball
linked with:
log4cplus 1.1.3
OpenSSL 1.0.2k-fips  26 Jan 2017
database:
Memfile backend 2.1

Running on CentOS 7.6.1810 VM kernel 3.10.0-957.1.3.el7.x86 with 2 * vCPU and 2GB RAM 

bellow is the dhcp6 config file omitting 102 subsets 

{
"Dhcp6": {
    "interfaces-config": {
        "interfaces": [ "ens192/1111:aef:aa77:83::100" ]
    },
    "mac-sources": [ "docsis-cmts", "client-link-addr-option" ],

    "control-socket": {
        "socket-type": "unix",
        "socket-name": "/tmp/kea-dhcp6-ctrl.sock"
    },

    "lease-database": {
        "type": "memfile",
        "lfc-interval": 3600
    },
    "expired-leases-processing": {
        "reclaim-timer-wait-time": 10,
        "flush-reclaimed-timer-wait-time": 25,
        "hold-reclaimed-time": 3600,
        "max-reclaim-leases": 100,
        "max-reclaim-time": 250,
        "unwarned-reclaim-cycles": 5
    },

    "renew-timer": 1000,
    "rebind-timer": 2000,
    "preferred-lifetime": 3000,
    "valid-lifetime": 4000,
    "option-data": [
        {
            "name": "dns-servers",
            "data": "1111:aef:aa77:1::22, 1111:aef:aa77:2::22"
        }
    ],
    "subnet6": [
        {
            "subnet": "1111:aec:0:10::/64",
            "pools": [ { "pool": "1111:aec:0:10::10-1111:aec:0:10:ffff::" } ],
            "pd-pools": [
                {
                    "prefix": "1111:aec:10::",
                    "prefix-len": 44,
                    "delegated-len": 56
                }
            ]
        },
//....
//  101 more subnets 
//....
 "hooks-libraries": [
        {
            "library": "/usr/local/lib/hooks/libdhcp_ha.so",
            "parameters": {
                "high-availability": [ {
                    "this-server-name": "KEA_DHCPv6_1",
                    "mode": "hot-standby",
                    "heartbeat-delay": 10000,
                    "max-response-delay": 20000,
                    "max-ack-delay": 10000,
                    "max-unacked-clients": 0,
                    "peers": [
                        {
                            "name": "KEA_DHCPv6_1",
                            "role": "primary",
                            "auto-failover": true
                        },
                        {
                            "name": "KEA_DHCPv6_2",
                            "role": "standby",
                            "auto-failover": true
                        }
                    ]
                } ]
            }
        }
    ]
},

"Logging":
{
  "loggers": [
    {
        "name": "kea-dhcp6",
        "output_options": [
            {
                "output": "/usr/local/var/log/kea-dhcp6.log"
            }
        ],
        "severity": "INFO",

        "debuglevel": 0
    }
  ]
}
}
Thanks

Nitzan

_______________________________________________
Kea-users mailing list
[hidden email]
https://lists.isc.org/mailman/listinfo/kea-users
Reply | Threaded
Open this post in threaded view
|

Re: [Kea-users] Fwd: Memory leak on dhcp6

Marcin Siodelski
Hi Nitzan,

We're trying to reproduce this issue using our test tools. If the leak
is related to the traffic volume processed by the server, it would be
useful for us to know the average number of packets/second that your
primary server is receiving.

You say that the memory consumption grows to 28% within an hour but I
don't know how many packets the server has received and processed to
reach that level. If you can provide the average DHCP traffic rate we
can craft the test that best mirrors your situation and confirm whether
or not we see the same thing.

Thanks in advance,

Marcin Siodelski
DHCP Software Engineer,
ISC

On 05/05/2019 16:29, Nitzan Tzelniker wrote:

> Hi,
>
> It seems that the issue is related to the HA
> After I commented the HA part the memory consumption is stayed low 
> I am adding the relevant configuration if you have any idea
>
> Thanks
>
> Nitzan
>
> Standby server ha configuration part: 
>
>     "hooks-libraries": [
>         {
>             "library": "/usr/local/lib/hooks/libdhcp_lease_cmds.so",
>             "parameters": { }
>         },
>         {
>             "library": "/usr/local/lib/hooks/libdhcp_ha.so",
>             "parameters": {
>                 "high-availability": [ {
>                     "this-server-name": "KEA_DHCPv6_2",
>                     "mode": "hot-standby",
>                     "heartbeat-delay": 10000,
>                     "max-response-delay": 20000,
>                     "max-ack-delay": 10000,
>                     "max-unacked-clients": 0,
>                     "peers": [
>                         {
>                             "name": "KEA_DHCPv6_1",
>                             "role": "primary",
>                             "auto-failover": true
>                         },
>                         {
>                             "name": "KEA_DHCPv6_2",
>                             "role": "standby",
>                             "auto-failover": true
>                         }
>                     ]
>                 } ]
>             }
>         }
>     ]
> }
>
> kea-ctrl-agent.conf  from both servers (Only the ip address is different ) 
>
> {
>
> "Control-agent": {
>     "http-host": "1.1.1.X",
>     "http-port": 8080,
>
>     "control-sockets": {
>         "dhcp4": {
>             "socket-type": "unix",
>             "socket-name": "/tmp/kea-dhcp4-ctrl.sock"
>         },
>         "dhcp6": {
>             "socket-type": "unix",
>             "socket-name": "/tmp/kea-dhcp6-ctrl.sock"
>         }
>     },
>
>     "hooks-libraries": [
>     ]
> },
>
> "Logging":
> {
>   "loggers": [
>     {
>         "name": "kea-ctrl-agent",
>         "output_options": [
>             {
>                 "output": "/usr/local/var/log/kea-ctrl-agent.log"
>             }
>         ],
>         "severity": "INFO",
>
>         "debuglevel": 0
>     }
>   ]
> }
> }
> ---------- Forwarded message ---------
> From: *Nitzan Tzelniker* <[hidden email]
> <mailto:[hidden email]>>
> Date: Fri, May 3, 2019 at 10:13 PM
> Subject: Memory leak on dhcp6
> To: <[hidden email] <mailto:[hidden email]>>
>
>
> Hi,
>
> I am running two dhcpv6 server with high-availability
> We have about 2500 regular leases and 2500 PD leases
> The setup is working but it look like kea-dhcp6 leak memory 
> When it start (with all of the leases in the file ) it take less then 1%
> of the memory but after two hours it take 28% and if I will give it 8
> hours it will crash 
>
> I don't see issue on the standby server only on the primary server
>
> Anybody saw this behavior  
> Any idea how to debug it
>
>  kea-dhcp6 -V
> 1.5.0
> tarball
> linked with:
> log4cplus 1.1.3
> OpenSSL 1.0.2k-fips  26 Jan 2017
> database:
> Memfile backend 2.1
>
> Running on CentOS 7.6.1810 VM kernel 3.10.0-957.1.3.el7.x86 with 2 *
> vCPU and 2GB RAM 
>
> bellow is the dhcp6 config file omitting 102 subsets 
>
> {
> "Dhcp6": {
>     "interfaces-config": {
>         "interfaces": [ "ens192/1111:aef:aa77:83::100" ]
>     },
>     "mac-sources": [ "docsis-cmts", "client-link-addr-option" ],
>
>     "control-socket": {
>         "socket-type": "unix",
>         "socket-name": "/tmp/kea-dhcp6-ctrl.sock"
>     },
>
>     "lease-database": {
>         "type": "memfile",
>         "lfc-interval": 3600
>     },
>     "expired-leases-processing": {
>         "reclaim-timer-wait-time": 10,
>         "flush-reclaimed-timer-wait-time": 25,
>         "hold-reclaimed-time": 3600,
>         "max-reclaim-leases": 100,
>         "max-reclaim-time": 250,
>         "unwarned-reclaim-cycles": 5
>     },
>
>     "renew-timer": 1000,
>     "rebind-timer": 2000,
>     "preferred-lifetime": 3000,
>     "valid-lifetime": 4000,
>     "option-data": [
>         {
>             "name": "dns-servers",
>             "data": "1111:aef:aa77:1::22, 1111:aef:aa77:2::22"
>         }
>     ],
>     "subnet6": [
>         {
>             "subnet": "1111:aec:0:10::/64",
>             "pools": [ { "pool":
> "1111:aec:0:10::10-1111:aec:0:10:ffff::" } ],
>             "pd-pools": [
>                 {
>                     "prefix": "1111:aec:10::",
>                     "prefix-len": 44,
>                     "delegated-len": 56
>                 }
>             ]
>         },
> //....
> //  101 more subnets 
> //....
>  "hooks-libraries": [
>         {
>             "library": "/usr/local/lib/hooks/libdhcp_ha.so",
>             "parameters": {
>                 "high-availability": [ {
>                     "this-server-name": "KEA_DHCPv6_1",
>                     "mode": "hot-standby",
>                     "heartbeat-delay": 10000,
>                     "max-response-delay": 20000,
>                     "max-ack-delay": 10000,
>                     "max-unacked-clients": 0,
>                     "peers": [
>                         {
>                             "name": "KEA_DHCPv6_1",
>                             "role": "primary",
>                             "auto-failover": true
>                         },
>                         {
>                             "name": "KEA_DHCPv6_2",
>                             "role": "standby",
>                             "auto-failover": true
>                         }
>                     ]
>                 } ]
>             }
>         }
>     ]
> },
>
> "Logging":
> {
>   "loggers": [
>     {
>         "name": "kea-dhcp6",
>         "output_options": [
>             {
>                 "output": "/usr/local/var/log/kea-dhcp6.log"
>             }
>         ],
>         "severity": "INFO",
>
>         "debuglevel": 0
>     }
>   ]
> }
> }
> Thanks
>
> Nitzan
>
> _______________________________________________
> Kea-users mailing list
> [hidden email]
> https://lists.isc.org/mailman/listinfo/kea-users
>

_______________________________________________
Kea-users mailing list
[hidden email]
https://lists.isc.org/mailman/listinfo/kea-users
Reply | Threaded
Open this post in threaded view
|

Re: [Kea-users] Fwd: Memory leak on dhcp6

Nitzan Tzelniker
Hi,

Just for anybody else that has this issue 
The issue was that I had the same server-id on both servers as we clone the VM of the first server after it create the server-id
I dont know why it cause memory leak.

Thenks

Nitzan


On Mon, May 6, 2019 at 6:56 PM Marcin Siodelski <[hidden email]> wrote:
Hi Nitzan,

We're trying to reproduce this issue using our test tools. If the leak
is related to the traffic volume processed by the server, it would be
useful for us to know the average number of packets/second that your
primary server is receiving.

You say that the memory consumption grows to 28% within an hour but I
don't know how many packets the server has received and processed to
reach that level. If you can provide the average DHCP traffic rate we
can craft the test that best mirrors your situation and confirm whether
or not we see the same thing.

Thanks in advance,

Marcin Siodelski
DHCP Software Engineer,
ISC

On 05/05/2019 16:29, Nitzan Tzelniker wrote:
> Hi,
>
> It seems that the issue is related to the HA
> After I commented the HA part the memory consumption is stayed low 
> I am adding the relevant configuration if you have any idea
>
> Thanks
>
> Nitzan
>
> Standby server ha configuration part: 
>
>     "hooks-libraries": [
>         {
>             "library": "/usr/local/lib/hooks/libdhcp_lease_cmds.so",
>             "parameters": { }
>         },
>         {
>             "library": "/usr/local/lib/hooks/libdhcp_ha.so",
>             "parameters": {
>                 "high-availability": [ {
>                     "this-server-name": "KEA_DHCPv6_2",
>                     "mode": "hot-standby",
>                     "heartbeat-delay": 10000,
>                     "max-response-delay": 20000,
>                     "max-ack-delay": 10000,
>                     "max-unacked-clients": 0,
>                     "peers": [
>                         {
>                             "name": "KEA_DHCPv6_1",
>                             "role": "primary",
>                             "auto-failover": true
>                         },
>                         {
>                             "name": "KEA_DHCPv6_2",
>                             "role": "standby",
>                             "auto-failover": true
>                         }
>                     ]
>                 } ]
>             }
>         }
>     ]
> }
>
> kea-ctrl-agent.conf  from both servers (Only the ip address is different ) 
>
> {
>
> "Control-agent": {
>     "http-host": "1.1.1.X",
>     "http-port": 8080,
>
>     "control-sockets": {
>         "dhcp4": {
>             "socket-type": "unix",
>             "socket-name": "/tmp/kea-dhcp4-ctrl.sock"
>         },
>         "dhcp6": {
>             "socket-type": "unix",
>             "socket-name": "/tmp/kea-dhcp6-ctrl.sock"
>         }
>     },
>
>     "hooks-libraries": [
>     ]
> },
>
> "Logging":
> {
>   "loggers": [
>     {
>         "name": "kea-ctrl-agent",
>         "output_options": [
>             {
>                 "output": "/usr/local/var/log/kea-ctrl-agent.log"
>             }
>         ],
>         "severity": "INFO",
>
>         "debuglevel": 0
>     }
>   ]
> }
> }
> ---------- Forwarded message ---------
> From: *Nitzan Tzelniker* <[hidden email]
> <mailto:[hidden email]>>
> Date: Fri, May 3, 2019 at 10:13 PM
> Subject: Memory leak on dhcp6
> To: <[hidden email] <mailto:[hidden email]>>
>
>
> Hi,
>
> I am running two dhcpv6 server with high-availability
> We have about 2500 regular leases and 2500 PD leases
> The setup is working but it look like kea-dhcp6 leak memory 
> When it start (with all of the leases in the file ) it take less then 1%
> of the memory but after two hours it take 28% and if I will give it 8
> hours it will crash 
>
> I don't see issue on the standby server only on the primary server
>
> Anybody saw this behavior  
> Any idea how to debug it
>
>  kea-dhcp6 -V
> 1.5.0
> tarball
> linked with:
> log4cplus 1.1.3
> OpenSSL 1.0.2k-fips  26 Jan 2017
> database:
> Memfile backend 2.1
>
> Running on CentOS 7.6.1810 VM kernel 3.10.0-957.1.3.el7.x86 with 2 *
> vCPU and 2GB RAM 
>
> bellow is the dhcp6 config file omitting 102 subsets 
>
> {
> "Dhcp6": {
>     "interfaces-config": {
>         "interfaces": [ "ens192/1111:aef:aa77:83::100" ]
>     },
>     "mac-sources": [ "docsis-cmts", "client-link-addr-option" ],
>
>     "control-socket": {
>         "socket-type": "unix",
>         "socket-name": "/tmp/kea-dhcp6-ctrl.sock"
>     },
>
>     "lease-database": {
>         "type": "memfile",
>         "lfc-interval": 3600
>     },
>     "expired-leases-processing": {
>         "reclaim-timer-wait-time": 10,
>         "flush-reclaimed-timer-wait-time": 25,
>         "hold-reclaimed-time": 3600,
>         "max-reclaim-leases": 100,
>         "max-reclaim-time": 250,
>         "unwarned-reclaim-cycles": 5
>     },
>
>     "renew-timer": 1000,
>     "rebind-timer": 2000,
>     "preferred-lifetime": 3000,
>     "valid-lifetime": 4000,
>     "option-data": [
>         {
>             "name": "dns-servers",
>             "data": "1111:aef:aa77:1::22, 1111:aef:aa77:2::22"
>         }
>     ],
>     "subnet6": [
>         {
>             "subnet": "1111:aec:0:10::/64",
>             "pools": [ { "pool":
> "1111:aec:0:10::10-1111:aec:0:10:ffff::" } ],
>             "pd-pools": [
>                 {
>                     "prefix": "1111:aec:10::",
>                     "prefix-len": 44,
>                     "delegated-len": 56
>                 }
>             ]
>         },
> //....
> //  101 more subnets 
> //....
>  "hooks-libraries": [
>         {
>             "library": "/usr/local/lib/hooks/libdhcp_ha.so",
>             "parameters": {
>                 "high-availability": [ {
>                     "this-server-name": "KEA_DHCPv6_1",
>                     "mode": "hot-standby",
>                     "heartbeat-delay": 10000,
>                     "max-response-delay": 20000,
>                     "max-ack-delay": 10000,
>                     "max-unacked-clients": 0,
>                     "peers": [
>                         {
>                             "name": "KEA_DHCPv6_1",
>                             "role": "primary",
>                             "auto-failover": true
>                         },
>                         {
>                             "name": "KEA_DHCPv6_2",
>                             "role": "standby",
>                             "auto-failover": true
>                         }
>                     ]
>                 } ]
>             }
>         }
>     ]
> },
>
> "Logging":
> {
>   "loggers": [
>     {
>         "name": "kea-dhcp6",
>         "output_options": [
>             {
>                 "output": "/usr/local/var/log/kea-dhcp6.log"
>             }
>         ],
>         "severity": "INFO",
>
>         "debuglevel": 0
>     }
>   ]
> }
> }
> Thanks
>
> Nitzan
>
> _______________________________________________
> Kea-users mailing list
> [hidden email]
> https://lists.isc.org/mailman/listinfo/kea-users
>


_______________________________________________
Kea-users mailing list
[hidden email]
https://lists.isc.org/mailman/listinfo/kea-users
Reply | Threaded
Open this post in threaded view
|

Re: [Kea-users] Fwd: Memory leak on dhcp6

Marcin Siodelski
Hello Nitzan,

Thank you for sharing this information. We'll try to reproduce this
behavior on our end. If the problem occurs, we'll investigate and fix it
because the duplicate server id is surely not a good reason for leaking
memory.... ;-)

Marcin Siodelski

DHCP Software Engineer,
ISC

On 07/05/2019 10:26, Nitzan Tzelniker wrote:

> Hi,
>
> Just for anybody else that has this issue 
> The issue was that I had the same server-id on both servers as we clone
> the VM of the first server after it create the server-id
> I dont know why it cause memory leak.
>
> Thenks
>
> Nitzan
>
>
> On Mon, May 6, 2019 at 6:56 PM Marcin Siodelski <[hidden email]
> <mailto:[hidden email]>> wrote:
>
>     Hi Nitzan,
>
>     We're trying to reproduce this issue using our test tools. If the leak
>     is related to the traffic volume processed by the server, it would be
>     useful for us to know the average number of packets/second that your
>     primary server is receiving.
>
>     You say that the memory consumption grows to 28% within an hour but I
>     don't know how many packets the server has received and processed to
>     reach that level. If you can provide the average DHCP traffic rate we
>     can craft the test that best mirrors your situation and confirm whether
>     or not we see the same thing.
>
>     Thanks in advance,
>
>     Marcin Siodelski
>     DHCP Software Engineer,
>     ISC
>
>     On 05/05/2019 16:29, Nitzan Tzelniker wrote:
>     > Hi,
>     >
>     > It seems that the issue is related to the HA
>     > After I commented the HA part the memory consumption is stayed low 
>     > I am adding the relevant configuration if you have any idea
>     >
>     > Thanks
>     >
>     > Nitzan
>     >
>     > Standby server ha configuration part: 
>     >
>     >     "hooks-libraries": [
>     >         {
>     >             "library": "/usr/local/lib/hooks/libdhcp_lease_cmds.so",
>     >             "parameters": { }
>     >         },
>     >         {
>     >             "library": "/usr/local/lib/hooks/libdhcp_ha.so",
>     >             "parameters": {
>     >                 "high-availability": [ {
>     >                     "this-server-name": "KEA_DHCPv6_2",
>     >                     "mode": "hot-standby",
>     >                     "heartbeat-delay": 10000,
>     >                     "max-response-delay": 20000,
>     >                     "max-ack-delay": 10000,
>     >                     "max-unacked-clients": 0,
>     >                     "peers": [
>     >                         {
>     >                             "name": "KEA_DHCPv6_1",
>     >                             "role": "primary",
>     >                             "auto-failover": true
>     >                         },
>     >                         {
>     >                             "name": "KEA_DHCPv6_2",
>     >                             "role": "standby",
>     >                             "auto-failover": true
>     >                         }
>     >                     ]
>     >                 } ]
>     >             }
>     >         }
>     >     ]
>     > }
>     >
>     > kea-ctrl-agent.conf  from both servers (Only the ip address is
>     different ) 
>     >
>     > {
>     >
>     > "Control-agent": {
>     >     "http-host": "1.1.1.X",
>     >     "http-port": 8080,
>     >
>     >     "control-sockets": {
>     >         "dhcp4": {
>     >             "socket-type": "unix",
>     >             "socket-name": "/tmp/kea-dhcp4-ctrl.sock"
>     >         },
>     >         "dhcp6": {
>     >             "socket-type": "unix",
>     >             "socket-name": "/tmp/kea-dhcp6-ctrl.sock"
>     >         }
>     >     },
>     >
>     >     "hooks-libraries": [
>     >     ]
>     > },
>     >
>     > "Logging":
>     > {
>     >   "loggers": [
>     >     {
>     >         "name": "kea-ctrl-agent",
>     >         "output_options": [
>     >             {
>     >                 "output": "/usr/local/var/log/kea-ctrl-agent.log"
>     >             }
>     >         ],
>     >         "severity": "INFO",
>     >
>     >         "debuglevel": 0
>     >     }
>     >   ]
>     > }
>     > }
>     > ---------- Forwarded message ---------
>     > From: *Nitzan Tzelniker* <[hidden email]
>     <mailto:[hidden email]>
>     > <mailto:[hidden email]
>     <mailto:[hidden email]>>>
>     > Date: Fri, May 3, 2019 at 10:13 PM
>     > Subject: Memory leak on dhcp6
>     > To: <[hidden email] <mailto:[hidden email]>
>     <mailto:[hidden email] <mailto:[hidden email]>>>
>     >
>     >
>     > Hi,
>     >
>     > I am running two dhcpv6 server with high-availability
>     > We have about 2500 regular leases and 2500 PD leases
>     > The setup is working but it look like kea-dhcp6 leak memory 
>     > When it start (with all of the leases in the file ) it take less
>     then 1%
>     > of the memory but after two hours it take 28% and if I will give it 8
>     > hours it will crash 
>     >
>     > I don't see issue on the standby server only on the primary server
>     >
>     > Anybody saw this behavior  
>     > Any idea how to debug it
>     >
>     >  kea-dhcp6 -V
>     > 1.5.0
>     > tarball
>     > linked with:
>     > log4cplus 1.1.3
>     > OpenSSL 1.0.2k-fips  26 Jan 2017
>     > database:
>     > Memfile backend 2.1
>     >
>     > Running on CentOS 7.6.1810 VM kernel 3.10.0-957.1.3.el7.x86 with 2 *
>     > vCPU and 2GB RAM 
>     >
>     > bellow is the dhcp6 config file omitting 102 subsets 
>     >
>     > {
>     > "Dhcp6": {
>     >     "interfaces-config": {
>     >         "interfaces": [ "ens192/1111:aef:aa77:83::100" ]
>     >     },
>     >     "mac-sources": [ "docsis-cmts", "client-link-addr-option" ],
>     >
>     >     "control-socket": {
>     >         "socket-type": "unix",
>     >         "socket-name": "/tmp/kea-dhcp6-ctrl.sock"
>     >     },
>     >
>     >     "lease-database": {
>     >         "type": "memfile",
>     >         "lfc-interval": 3600
>     >     },
>     >     "expired-leases-processing": {
>     >         "reclaim-timer-wait-time": 10,
>     >         "flush-reclaimed-timer-wait-time": 25,
>     >         "hold-reclaimed-time": 3600,
>     >         "max-reclaim-leases": 100,
>     >         "max-reclaim-time": 250,
>     >         "unwarned-reclaim-cycles": 5
>     >     },
>     >
>     >     "renew-timer": 1000,
>     >     "rebind-timer": 2000,
>     >     "preferred-lifetime": 3000,
>     >     "valid-lifetime": 4000,
>     >     "option-data": [
>     >         {
>     >             "name": "dns-servers",
>     >             "data": "1111:aef:aa77:1::22, 1111:aef:aa77:2::22"
>     >         }
>     >     ],
>     >     "subnet6": [
>     >         {
>     >             "subnet": "1111:aec:0:10::/64",
>     >             "pools": [ { "pool":
>     > "1111:aec:0:10::10-1111:aec:0:10:ffff::" } ],
>     >             "pd-pools": [
>     >                 {
>     >                     "prefix": "1111:aec:10::",
>     >                     "prefix-len": 44,
>     >                     "delegated-len": 56
>     >                 }
>     >             ]
>     >         },
>     > //....
>     > //  101 more subnets 
>     > //....
>     >  "hooks-libraries": [
>     >         {
>     >             "library": "/usr/local/lib/hooks/libdhcp_ha.so",
>     >             "parameters": {
>     >                 "high-availability": [ {
>     >                     "this-server-name": "KEA_DHCPv6_1",
>     >                     "mode": "hot-standby",
>     >                     "heartbeat-delay": 10000,
>     >                     "max-response-delay": 20000,
>     >                     "max-ack-delay": 10000,
>     >                     "max-unacked-clients": 0,
>     >                     "peers": [
>     >                         {
>     >                             "name": "KEA_DHCPv6_1",
>     >                             "role": "primary",
>     >                             "auto-failover": true
>     >                         },
>     >                         {
>     >                             "name": "KEA_DHCPv6_2",
>     >                             "role": "standby",
>     >                             "auto-failover": true
>     >                         }
>     >                     ]
>     >                 } ]
>     >             }
>     >         }
>     >     ]
>     > },
>     >
>     > "Logging":
>     > {
>     >   "loggers": [
>     >     {
>     >         "name": "kea-dhcp6",
>     >         "output_options": [
>     >             {
>     >                 "output": "/usr/local/var/log/kea-dhcp6.log"
>     >             }
>     >         ],
>     >         "severity": "INFO",
>     >
>     >         "debuglevel": 0
>     >     }
>     >   ]
>     > }
>     > }
>     > Thanks
>     >
>     > Nitzan
>     >
>     > _______________________________________________
>     > Kea-users mailing list
>     > [hidden email] <mailto:[hidden email]>
>     > https://lists.isc.org/mailman/listinfo/kea-users
>     >
>

_______________________________________________
Kea-users mailing list
[hidden email]
https://lists.isc.org/mailman/listinfo/kea-users
Reply | Threaded
Open this post in threaded view
|

Re: [Kea-users] Fwd: Memory leak on dhcp6

Nitzan Tzelniker
Thanks for your support. 
Unless I miss it in the logs it might also be a good idea to have error message if both servers have the same server-id 

Thanks

Nitzan

On Tue, May 7, 2019 at 11:45 AM Marcin Siodelski <[hidden email]> wrote:
Hello Nitzan,

Thank you for sharing this information. We'll try to reproduce this
behavior on our end. If the problem occurs, we'll investigate and fix it
because the duplicate server id is surely not a good reason for leaking
memory.... ;-)

Marcin Siodelski

DHCP Software Engineer,
ISC

On 07/05/2019 10:26, Nitzan Tzelniker wrote:
> Hi,
>
> Just for anybody else that has this issue 
> The issue was that I had the same server-id on both servers as we clone
> the VM of the first server after it create the server-id
> I dont know why it cause memory leak.
>
> Thenks
>
> Nitzan
>
>
> On Mon, May 6, 2019 at 6:56 PM Marcin Siodelski <[hidden email]
> <mailto:[hidden email]>> wrote:
>
>     Hi Nitzan,
>
>     We're trying to reproduce this issue using our test tools. If the leak
>     is related to the traffic volume processed by the server, it would be
>     useful for us to know the average number of packets/second that your
>     primary server is receiving.
>
>     You say that the memory consumption grows to 28% within an hour but I
>     don't know how many packets the server has received and processed to
>     reach that level. If you can provide the average DHCP traffic rate we
>     can craft the test that best mirrors your situation and confirm whether
>     or not we see the same thing.
>
>     Thanks in advance,
>
>     Marcin Siodelski
>     DHCP Software Engineer,
>     ISC
>
>     On 05/05/2019 16:29, Nitzan Tzelniker wrote:
>     > Hi,
>     >
>     > It seems that the issue is related to the HA
>     > After I commented the HA part the memory consumption is stayed low 
>     > I am adding the relevant configuration if you have any idea
>     >
>     > Thanks
>     >
>     > Nitzan
>     >
>     > Standby server ha configuration part: 
>     >
>     >     "hooks-libraries": [
>     >         {
>     >             "library": "/usr/local/lib/hooks/libdhcp_lease_cmds.so",
>     >             "parameters": { }
>     >         },
>     >         {
>     >             "library": "/usr/local/lib/hooks/libdhcp_ha.so",
>     >             "parameters": {
>     >                 "high-availability": [ {
>     >                     "this-server-name": "KEA_DHCPv6_2",
>     >                     "mode": "hot-standby",
>     >                     "heartbeat-delay": 10000,
>     >                     "max-response-delay": 20000,
>     >                     "max-ack-delay": 10000,
>     >                     "max-unacked-clients": 0,
>     >                     "peers": [
>     >                         {
>     >                             "name": "KEA_DHCPv6_1",
>     >                             "role": "primary",
>     >                             "auto-failover": true
>     >                         },
>     >                         {
>     >                             "name": "KEA_DHCPv6_2",
>     >                             "role": "standby",
>     >                             "auto-failover": true
>     >                         }
>     >                     ]
>     >                 } ]
>     >             }
>     >         }
>     >     ]
>     > }
>     >
>     > kea-ctrl-agent.conf  from both servers (Only the ip address is
>     different ) 
>     >
>     > {
>     >
>     > "Control-agent": {
>     >     "http-host": "1.1.1.X",
>     >     "http-port": 8080,
>     >
>     >     "control-sockets": {
>     >         "dhcp4": {
>     >             "socket-type": "unix",
>     >             "socket-name": "/tmp/kea-dhcp4-ctrl.sock"
>     >         },
>     >         "dhcp6": {
>     >             "socket-type": "unix",
>     >             "socket-name": "/tmp/kea-dhcp6-ctrl.sock"
>     >         }
>     >     },
>     >
>     >     "hooks-libraries": [
>     >     ]
>     > },
>     >
>     > "Logging":
>     > {
>     >   "loggers": [
>     >     {
>     >         "name": "kea-ctrl-agent",
>     >         "output_options": [
>     >             {
>     >                 "output": "/usr/local/var/log/kea-ctrl-agent.log"
>     >             }
>     >         ],
>     >         "severity": "INFO",
>     >
>     >         "debuglevel": 0
>     >     }
>     >   ]
>     > }
>     > }
>     > ---------- Forwarded message ---------
>     > From: *Nitzan Tzelniker* <[hidden email]
>     <mailto:[hidden email]>
>     > <mailto:[hidden email]
>     <mailto:[hidden email]>>>
>     > Date: Fri, May 3, 2019 at 10:13 PM
>     > Subject: Memory leak on dhcp6
>     > To: <[hidden email] <mailto:[hidden email]>
>     <mailto:[hidden email] <mailto:[hidden email]>>>
>     >
>     >
>     > Hi,
>     >
>     > I am running two dhcpv6 server with high-availability
>     > We have about 2500 regular leases and 2500 PD leases
>     > The setup is working but it look like kea-dhcp6 leak memory 
>     > When it start (with all of the leases in the file ) it take less
>     then 1%
>     > of the memory but after two hours it take 28% and if I will give it 8
>     > hours it will crash 
>     >
>     > I don't see issue on the standby server only on the primary server
>     >
>     > Anybody saw this behavior  
>     > Any idea how to debug it
>     >
>     >  kea-dhcp6 -V
>     > 1.5.0
>     > tarball
>     > linked with:
>     > log4cplus 1.1.3
>     > OpenSSL 1.0.2k-fips  26 Jan 2017
>     > database:
>     > Memfile backend 2.1
>     >
>     > Running on CentOS 7.6.1810 VM kernel 3.10.0-957.1.3.el7.x86 with 2 *
>     > vCPU and 2GB RAM 
>     >
>     > bellow is the dhcp6 config file omitting 102 subsets 
>     >
>     > {
>     > "Dhcp6": {
>     >     "interfaces-config": {
>     >         "interfaces": [ "ens192/1111:aef:aa77:83::100" ]
>     >     },
>     >     "mac-sources": [ "docsis-cmts", "client-link-addr-option" ],
>     >
>     >     "control-socket": {
>     >         "socket-type": "unix",
>     >         "socket-name": "/tmp/kea-dhcp6-ctrl.sock"
>     >     },
>     >
>     >     "lease-database": {
>     >         "type": "memfile",
>     >         "lfc-interval": 3600
>     >     },
>     >     "expired-leases-processing": {
>     >         "reclaim-timer-wait-time": 10,
>     >         "flush-reclaimed-timer-wait-time": 25,
>     >         "hold-reclaimed-time": 3600,
>     >         "max-reclaim-leases": 100,
>     >         "max-reclaim-time": 250,
>     >         "unwarned-reclaim-cycles": 5
>     >     },
>     >
>     >     "renew-timer": 1000,
>     >     "rebind-timer": 2000,
>     >     "preferred-lifetime": 3000,
>     >     "valid-lifetime": 4000,
>     >     "option-data": [
>     >         {
>     >             "name": "dns-servers",
>     >             "data": "1111:aef:aa77:1::22, 1111:aef:aa77:2::22"
>     >         }
>     >     ],
>     >     "subnet6": [
>     >         {
>     >             "subnet": "1111:aec:0:10::/64",
>     >             "pools": [ { "pool":
>     > "1111:aec:0:10::10-1111:aec:0:10:ffff::" } ],
>     >             "pd-pools": [
>     >                 {
>     >                     "prefix": "1111:aec:10::",
>     >                     "prefix-len": 44,
>     >                     "delegated-len": 56
>     >                 }
>     >             ]
>     >         },
>     > //....
>     > //  101 more subnets 
>     > //....
>     >  "hooks-libraries": [
>     >         {
>     >             "library": "/usr/local/lib/hooks/libdhcp_ha.so",
>     >             "parameters": {
>     >                 "high-availability": [ {
>     >                     "this-server-name": "KEA_DHCPv6_1",
>     >                     "mode": "hot-standby",
>     >                     "heartbeat-delay": 10000,
>     >                     "max-response-delay": 20000,
>     >                     "max-ack-delay": 10000,
>     >                     "max-unacked-clients": 0,
>     >                     "peers": [
>     >                         {
>     >                             "name": "KEA_DHCPv6_1",
>     >                             "role": "primary",
>     >                             "auto-failover": true
>     >                         },
>     >                         {
>     >                             "name": "KEA_DHCPv6_2",
>     >                             "role": "standby",
>     >                             "auto-failover": true
>     >                         }
>     >                     ]
>     >                 } ]
>     >             }
>     >         }
>     >     ]
>     > },
>     >
>     > "Logging":
>     > {
>     >   "loggers": [
>     >     {
>     >         "name": "kea-dhcp6",
>     >         "output_options": [
>     >             {
>     >                 "output": "/usr/local/var/log/kea-dhcp6.log"
>     >             }
>     >         ],
>     >         "severity": "INFO",
>     >
>     >         "debuglevel": 0
>     >     }
>     >   ]
>     > }
>     > }
>     > Thanks
>     >
>     > Nitzan
>     >
>     > _______________________________________________
>     > Kea-users mailing list
>     > [hidden email] <mailto:[hidden email]>
>     > https://lists.isc.org/mailman/listinfo/kea-users
>     >
>


_______________________________________________
Kea-users mailing list
[hidden email]
https://lists.isc.org/mailman/listinfo/kea-users
Reply | Threaded
Open this post in threaded view
|

Re: [Kea-users] Fwd: Memory leak on dhcp6

Marcin Siodelski
Nitzan,

I just wanted to let you know that so far we were unable to reproduce
the memory leak with duplicated server-id across the HA pair. Our test
did not include prefix delegation, only IA_NAs. Are your clients
requesting both address assignment and prefix delegation in the single
transaction?

Thanks,

Marcin

On 07/05/2019 10:49, Nitzan Tzelniker wrote:

> Thanks for your support. 
> Unless I miss it in the logs it might also be a good idea to have error
> message if both servers have the same server-id 
>
> Thanks
>
> Nitzan
>
> On Tue, May 7, 2019 at 11:45 AM Marcin Siodelski <[hidden email]
> <mailto:[hidden email]>> wrote:
>
>     Hello Nitzan,
>
>     Thank you for sharing this information. We'll try to reproduce this
>     behavior on our end. If the problem occurs, we'll investigate and fix it
>     because the duplicate server id is surely not a good reason for leaking
>     memory.... ;-)
>
>     Marcin Siodelski
>
>     DHCP Software Engineer,
>     ISC
>
>     On 07/05/2019 10:26, Nitzan Tzelniker wrote:
>     > Hi,
>     >
>     > Just for anybody else that has this issue 
>     > The issue was that I had the same server-id on both servers as we
>     clone
>     > the VM of the first server after it create the server-id
>     > I dont know why it cause memory leak.
>     >
>     > Thenks
>     >
>     > Nitzan
>     >
>     >
>     > On Mon, May 6, 2019 at 6:56 PM Marcin Siodelski <[hidden email]
>     <mailto:[hidden email]>
>     > <mailto:[hidden email] <mailto:[hidden email]>>> wrote:
>     >
>     >     Hi Nitzan,
>     >
>     >     We're trying to reproduce this issue using our test tools. If
>     the leak
>     >     is related to the traffic volume processed by the server, it
>     would be
>     >     useful for us to know the average number of packets/second
>     that your
>     >     primary server is receiving.
>     >
>     >     You say that the memory consumption grows to 28% within an
>     hour but I
>     >     don't know how many packets the server has received and
>     processed to
>     >     reach that level. If you can provide the average DHCP traffic
>     rate we
>     >     can craft the test that best mirrors your situation and
>     confirm whether
>     >     or not we see the same thing.
>     >
>     >     Thanks in advance,
>     >
>     >     Marcin Siodelski
>     >     DHCP Software Engineer,
>     >     ISC
>     >
>     >     On 05/05/2019 16:29, Nitzan Tzelniker wrote:
>     >     > Hi,
>     >     >
>     >     > It seems that the issue is related to the HA
>     >     > After I commented the HA part the memory consumption is
>     stayed low 
>     >     > I am adding the relevant configuration if you have any idea
>     >     >
>     >     > Thanks
>     >     >
>     >     > Nitzan
>     >     >
>     >     > Standby server ha configuration part: 
>     >     >
>     >     >     "hooks-libraries": [
>     >     >         {
>     >     >             "library":
>     "/usr/local/lib/hooks/libdhcp_lease_cmds.so",
>     >     >             "parameters": { }
>     >     >         },
>     >     >         {
>     >     >             "library": "/usr/local/lib/hooks/libdhcp_ha.so",
>     >     >             "parameters": {
>     >     >                 "high-availability": [ {
>     >     >                     "this-server-name": "KEA_DHCPv6_2",
>     >     >                     "mode": "hot-standby",
>     >     >                     "heartbeat-delay": 10000,
>     >     >                     "max-response-delay": 20000,
>     >     >                     "max-ack-delay": 10000,
>     >     >                     "max-unacked-clients": 0,
>     >     >                     "peers": [
>     >     >                         {
>     >     >                             "name": "KEA_DHCPv6_1",
>     >     >                             "role": "primary",
>     >     >                             "auto-failover": true
>     >     >                         },
>     >     >                         {
>     >     >                             "name": "KEA_DHCPv6_2",
>     >     >                             "role": "standby",
>     >     >                             "auto-failover": true
>     >     >                         }
>     >     >                     ]
>     >     >                 } ]
>     >     >             }
>     >     >         }
>     >     >     ]
>     >     > }
>     >     >
>     >     > kea-ctrl-agent.conf  from both servers (Only the ip address is
>     >     different ) 
>     >     >
>     >     > {
>     >     >
>     >     > "Control-agent": {
>     >     >     "http-host": "1.1.1.X",
>     >     >     "http-port": 8080,
>     >     >
>     >     >     "control-sockets": {
>     >     >         "dhcp4": {
>     >     >             "socket-type": "unix",
>     >     >             "socket-name": "/tmp/kea-dhcp4-ctrl.sock"
>     >     >         },
>     >     >         "dhcp6": {
>     >     >             "socket-type": "unix",
>     >     >             "socket-name": "/tmp/kea-dhcp6-ctrl.sock"
>     >     >         }
>     >     >     },
>     >     >
>     >     >     "hooks-libraries": [
>     >     >     ]
>     >     > },
>     >     >
>     >     > "Logging":
>     >     > {
>     >     >   "loggers": [
>     >     >     {
>     >     >         "name": "kea-ctrl-agent",
>     >     >         "output_options": [
>     >     >             {
>     >     >                 "output":
>     "/usr/local/var/log/kea-ctrl-agent.log"
>     >     >             }
>     >     >         ],
>     >     >         "severity": "INFO",
>     >     >
>     >     >         "debuglevel": 0
>     >     >     }
>     >     >   ]
>     >     > }
>     >     > }
>     >     > ---------- Forwarded message ---------
>     >     > From: *Nitzan Tzelniker* <[hidden email]
>     <mailto:[hidden email]>
>     >     <mailto:[hidden email]
>     <mailto:[hidden email]>>
>     >     > <mailto:[hidden email]
>     <mailto:[hidden email]>
>     >     <mailto:[hidden email]
>     <mailto:[hidden email]>>>>
>     >     > Date: Fri, May 3, 2019 at 10:13 PM
>     >     > Subject: Memory leak on dhcp6
>     >     > To: <[hidden email]
>     <mailto:[hidden email]> <mailto:[hidden email]
>     <mailto:[hidden email]>>
>     >     <mailto:[hidden email]
>     <mailto:[hidden email]> <mailto:[hidden email]
>     <mailto:[hidden email]>>>>
>     >     >
>     >     >
>     >     > Hi,
>     >     >
>     >     > I am running two dhcpv6 server with high-availability
>     >     > We have about 2500 regular leases and 2500 PD leases
>     >     > The setup is working but it look like kea-dhcp6 leak memory 
>     >     > When it start (with all of the leases in the file ) it take less
>     >     then 1%
>     >     > of the memory but after two hours it take 28% and if I will
>     give it 8
>     >     > hours it will crash 
>     >     >
>     >     > I don't see issue on the standby server only on the primary
>     server
>     >     >
>     >     > Anybody saw this behavior  
>     >     > Any idea how to debug it
>     >     >
>     >     >  kea-dhcp6 -V
>     >     > 1.5.0
>     >     > tarball
>     >     > linked with:
>     >     > log4cplus 1.1.3
>     >     > OpenSSL 1.0.2k-fips  26 Jan 2017
>     >     > database:
>     >     > Memfile backend 2.1
>     >     >
>     >     > Running on CentOS 7.6.1810 VM kernel 3.10.0-957.1.3.el7.x86
>     with 2 *
>     >     > vCPU and 2GB RAM 
>     >     >
>     >     > bellow is the dhcp6 config file omitting 102 subsets 
>     >     >
>     >     > {
>     >     > "Dhcp6": {
>     >     >     "interfaces-config": {
>     >     >         "interfaces": [ "ens192/1111:aef:aa77:83::100" ]
>     >     >     },
>     >     >     "mac-sources": [ "docsis-cmts", "client-link-addr-option" ],
>     >     >
>     >     >     "control-socket": {
>     >     >         "socket-type": "unix",
>     >     >         "socket-name": "/tmp/kea-dhcp6-ctrl.sock"
>     >     >     },
>     >     >
>     >     >     "lease-database": {
>     >     >         "type": "memfile",
>     >     >         "lfc-interval": 3600
>     >     >     },
>     >     >     "expired-leases-processing": {
>     >     >         "reclaim-timer-wait-time": 10,
>     >     >         "flush-reclaimed-timer-wait-time": 25,
>     >     >         "hold-reclaimed-time": 3600,
>     >     >         "max-reclaim-leases": 100,
>     >     >         "max-reclaim-time": 250,
>     >     >         "unwarned-reclaim-cycles": 5
>     >     >     },
>     >     >
>     >     >     "renew-timer": 1000,
>     >     >     "rebind-timer": 2000,
>     >     >     "preferred-lifetime": 3000,
>     >     >     "valid-lifetime": 4000,
>     >     >     "option-data": [
>     >     >         {
>     >     >             "name": "dns-servers",
>     >     >             "data": "1111:aef:aa77:1::22, 1111:aef:aa77:2::22"
>     >     >         }
>     >     >     ],
>     >     >     "subnet6": [
>     >     >         {
>     >     >             "subnet": "1111:aec:0:10::/64",
>     >     >             "pools": [ { "pool":
>     >     > "1111:aec:0:10::10-1111:aec:0:10:ffff::" } ],
>     >     >             "pd-pools": [
>     >     >                 {
>     >     >                     "prefix": "1111:aec:10::",
>     >     >                     "prefix-len": 44,
>     >     >                     "delegated-len": 56
>     >     >                 }
>     >     >             ]
>     >     >         },
>     >     > //....
>     >     > //  101 more subnets 
>     >     > //....
>     >     >  "hooks-libraries": [
>     >     >         {
>     >     >             "library": "/usr/local/lib/hooks/libdhcp_ha.so",
>     >     >             "parameters": {
>     >     >                 "high-availability": [ {
>     >     >                     "this-server-name": "KEA_DHCPv6_1",
>     >     >                     "mode": "hot-standby",
>     >     >                     "heartbeat-delay": 10000,
>     >     >                     "max-response-delay": 20000,
>     >     >                     "max-ack-delay": 10000,
>     >     >                     "max-unacked-clients": 0,
>     >     >                     "peers": [
>     >     >                         {
>     >     >                             "name": "KEA_DHCPv6_1",
>     >     >                             "role": "primary",
>     >     >                             "auto-failover": true
>     >     >                         },
>     >     >                         {
>     >     >                             "name": "KEA_DHCPv6_2",
>     >     >                             "role": "standby",
>     >     >                             "auto-failover": true
>     >     >                         }
>     >     >                     ]
>     >     >                 } ]
>     >     >             }
>     >     >         }
>     >     >     ]
>     >     > },
>     >     >
>     >     > "Logging":
>     >     > {
>     >     >   "loggers": [
>     >     >     {
>     >     >         "name": "kea-dhcp6",
>     >     >         "output_options": [
>     >     >             {
>     >     >                 "output": "/usr/local/var/log/kea-dhcp6.log"
>     >     >             }
>     >     >         ],
>     >     >         "severity": "INFO",
>     >     >
>     >     >         "debuglevel": 0
>     >     >     }
>     >     >   ]
>     >     > }
>     >     > }
>     >     > Thanks
>     >     >
>     >     > Nitzan
>     >     >
>     >     > _______________________________________________
>     >     > Kea-users mailing list
>     >     > [hidden email] <mailto:[hidden email]>
>     <mailto:[hidden email] <mailto:[hidden email]>>
>     >     > https://lists.isc.org/mailman/listinfo/kea-users
>     >     >
>     >
>

_______________________________________________
Kea-users mailing list
[hidden email]
https://lists.isc.org/mailman/listinfo/kea-users
Reply | Threaded
Open this post in threaded view
|

Re: [Kea-users] Fwd: Memory leak on dhcp6

Nitzan Tzelniker
Hi Marcin,

As far as I see in the PCAP most of the customer ask in separate packets but I see few that ask for both in a single packet 
If it will help I took a coredump during the issue 

Regards

Nitzan

On Wed, May 8, 2019 at 4:51 PM Marcin Siodelski <[hidden email]> wrote:
Nitzan,

I just wanted to let you know that so far we were unable to reproduce
the memory leak with duplicated server-id across the HA pair. Our test
did not include prefix delegation, only IA_NAs. Are your clients
requesting both address assignment and prefix delegation in the single
transaction?

Thanks,

Marcin

On 07/05/2019 10:49, Nitzan Tzelniker wrote:
> Thanks for your support. 
> Unless I miss it in the logs it might also be a good idea to have error
> message if both servers have the same server-id 
>
> Thanks
>
> Nitzan
>
> On Tue, May 7, 2019 at 11:45 AM Marcin Siodelski <[hidden email]
> <mailto:[hidden email]>> wrote:
>
>     Hello Nitzan,
>
>     Thank you for sharing this information. We'll try to reproduce this
>     behavior on our end. If the problem occurs, we'll investigate and fix it
>     because the duplicate server id is surely not a good reason for leaking
>     memory.... ;-)
>
>     Marcin Siodelski
>
>     DHCP Software Engineer,
>     ISC
>
>     On 07/05/2019 10:26, Nitzan Tzelniker wrote:
>     > Hi,
>     >
>     > Just for anybody else that has this issue 
>     > The issue was that I had the same server-id on both servers as we
>     clone
>     > the VM of the first server after it create the server-id
>     > I dont know why it cause memory leak.
>     >
>     > Thenks
>     >
>     > Nitzan
>     >
>     >
>     > On Mon, May 6, 2019 at 6:56 PM Marcin Siodelski <[hidden email]
>     <mailto:[hidden email]>
>     > <mailto:[hidden email] <mailto:[hidden email]>>> wrote:
>     >
>     >     Hi Nitzan,
>     >
>     >     We're trying to reproduce this issue using our test tools. If
>     the leak
>     >     is related to the traffic volume processed by the server, it
>     would be
>     >     useful for us to know the average number of packets/second
>     that your
>     >     primary server is receiving.
>     >
>     >     You say that the memory consumption grows to 28% within an
>     hour but I
>     >     don't know how many packets the server has received and
>     processed to
>     >     reach that level. If you can provide the average DHCP traffic
>     rate we
>     >     can craft the test that best mirrors your situation and
>     confirm whether
>     >     or not we see the same thing.
>     >
>     >     Thanks in advance,
>     >
>     >     Marcin Siodelski
>     >     DHCP Software Engineer,
>     >     ISC
>     >
>     >     On 05/05/2019 16:29, Nitzan Tzelniker wrote:
>     >     > Hi,
>     >     >
>     >     > It seems that the issue is related to the HA
>     >     > After I commented the HA part the memory consumption is
>     stayed low 
>     >     > I am adding the relevant configuration if you have any idea
>     >     >
>     >     > Thanks
>     >     >
>     >     > Nitzan
>     >     >
>     >     > Standby server ha configuration part: 
>     >     >
>     >     >     "hooks-libraries": [
>     >     >         {
>     >     >             "library":
>     "/usr/local/lib/hooks/libdhcp_lease_cmds.so",
>     >     >             "parameters": { }
>     >     >         },
>     >     >         {
>     >     >             "library": "/usr/local/lib/hooks/libdhcp_ha.so",
>     >     >             "parameters": {
>     >     >                 "high-availability": [ {
>     >     >                     "this-server-name": "KEA_DHCPv6_2",
>     >     >                     "mode": "hot-standby",
>     >     >                     "heartbeat-delay": 10000,
>     >     >                     "max-response-delay": 20000,
>     >     >                     "max-ack-delay": 10000,
>     >     >                     "max-unacked-clients": 0,
>     >     >                     "peers": [
>     >     >                         {
>     >     >                             "name": "KEA_DHCPv6_1",
>     >     >                             "role": "primary",
>     >     >                             "auto-failover": true
>     >     >                         },
>     >     >                         {
>     >     >                             "name": "KEA_DHCPv6_2",
>     >     >                             "role": "standby",
>     >     >                             "auto-failover": true
>     >     >                         }
>     >     >                     ]
>     >     >                 } ]
>     >     >             }
>     >     >         }
>     >     >     ]
>     >     > }
>     >     >
>     >     > kea-ctrl-agent.conf  from both servers (Only the ip address is
>     >     different ) 
>     >     >
>     >     > {
>     >     >
>     >     > "Control-agent": {
>     >     >     "http-host": "1.1.1.X",
>     >     >     "http-port": 8080,
>     >     >
>     >     >     "control-sockets": {
>     >     >         "dhcp4": {
>     >     >             "socket-type": "unix",
>     >     >             "socket-name": "/tmp/kea-dhcp4-ctrl.sock"
>     >     >         },
>     >     >         "dhcp6": {
>     >     >             "socket-type": "unix",
>     >     >             "socket-name": "/tmp/kea-dhcp6-ctrl.sock"
>     >     >         }
>     >     >     },
>     >     >
>     >     >     "hooks-libraries": [
>     >     >     ]
>     >     > },
>     >     >
>     >     > "Logging":
>     >     > {
>     >     >   "loggers": [
>     >     >     {
>     >     >         "name": "kea-ctrl-agent",
>     >     >         "output_options": [
>     >     >             {
>     >     >                 "output":
>     "/usr/local/var/log/kea-ctrl-agent.log"
>     >     >             }
>     >     >         ],
>     >     >         "severity": "INFO",
>     >     >
>     >     >         "debuglevel": 0
>     >     >     }
>     >     >   ]
>     >     > }
>     >     > }
>     >     > ---------- Forwarded message ---------
>     >     > From: *Nitzan Tzelniker* <[hidden email]
>     <mailto:[hidden email]>
>     >     <mailto:[hidden email]
>     <mailto:[hidden email]>>
>     >     > <mailto:[hidden email]
>     <mailto:[hidden email]>
>     >     <mailto:[hidden email]
>     <mailto:[hidden email]>>>>
>     >     > Date: Fri, May 3, 2019 at 10:13 PM
>     >     > Subject: Memory leak on dhcp6
>     >     > To: <[hidden email]
>     <mailto:[hidden email]> <mailto:[hidden email]
>     <mailto:[hidden email]>>
>     >     <mailto:[hidden email]
>     <mailto:[hidden email]> <mailto:[hidden email]
>     <mailto:[hidden email]>>>>
>     >     >
>     >     >
>     >     > Hi,
>     >     >
>     >     > I am running two dhcpv6 server with high-availability
>     >     > We have about 2500 regular leases and 2500 PD leases
>     >     > The setup is working but it look like kea-dhcp6 leak memory 
>     >     > When it start (with all of the leases in the file ) it take less
>     >     then 1%
>     >     > of the memory but after two hours it take 28% and if I will
>     give it 8
>     >     > hours it will crash 
>     >     >
>     >     > I don't see issue on the standby server only on the primary
>     server
>     >     >
>     >     > Anybody saw this behavior  
>     >     > Any idea how to debug it
>     >     >
>     >     >  kea-dhcp6 -V
>     >     > 1.5.0
>     >     > tarball
>     >     > linked with:
>     >     > log4cplus 1.1.3
>     >     > OpenSSL 1.0.2k-fips  26 Jan 2017
>     >     > database:
>     >     > Memfile backend 2.1
>     >     >
>     >     > Running on CentOS 7.6.1810 VM kernel 3.10.0-957.1.3.el7.x86
>     with 2 *
>     >     > vCPU and 2GB RAM 
>     >     >
>     >     > bellow is the dhcp6 config file omitting 102 subsets 
>     >     >
>     >     > {
>     >     > "Dhcp6": {
>     >     >     "interfaces-config": {
>     >     >         "interfaces": [ "ens192/1111:aef:aa77:83::100" ]
>     >     >     },
>     >     >     "mac-sources": [ "docsis-cmts", "client-link-addr-option" ],
>     >     >
>     >     >     "control-socket": {
>     >     >         "socket-type": "unix",
>     >     >         "socket-name": "/tmp/kea-dhcp6-ctrl.sock"
>     >     >     },
>     >     >
>     >     >     "lease-database": {
>     >     >         "type": "memfile",
>     >     >         "lfc-interval": 3600
>     >     >     },
>     >     >     "expired-leases-processing": {
>     >     >         "reclaim-timer-wait-time": 10,
>     >     >         "flush-reclaimed-timer-wait-time": 25,
>     >     >         "hold-reclaimed-time": 3600,
>     >     >         "max-reclaim-leases": 100,
>     >     >         "max-reclaim-time": 250,
>     >     >         "unwarned-reclaim-cycles": 5
>     >     >     },
>     >     >
>     >     >     "renew-timer": 1000,
>     >     >     "rebind-timer": 2000,
>     >     >     "preferred-lifetime": 3000,
>     >     >     "valid-lifetime": 4000,
>     >     >     "option-data": [
>     >     >         {
>     >     >             "name": "dns-servers",
>     >     >             "data": "1111:aef:aa77:1::22, 1111:aef:aa77:2::22"
>     >     >         }
>     >     >     ],
>     >     >     "subnet6": [
>     >     >         {
>     >     >             "subnet": "1111:aec:0:10::/64",
>     >     >             "pools": [ { "pool":
>     >     > "1111:aec:0:10::10-1111:aec:0:10:ffff::" } ],
>     >     >             "pd-pools": [
>     >     >                 {
>     >     >                     "prefix": "1111:aec:10::",
>     >     >                     "prefix-len": 44,
>     >     >                     "delegated-len": 56
>     >     >                 }
>     >     >             ]
>     >     >         },
>     >     > //....
>     >     > //  101 more subnets 
>     >     > //....
>     >     >  "hooks-libraries": [
>     >     >         {
>     >     >             "library": "/usr/local/lib/hooks/libdhcp_ha.so",
>     >     >             "parameters": {
>     >     >                 "high-availability": [ {
>     >     >                     "this-server-name": "KEA_DHCPv6_1",
>     >     >                     "mode": "hot-standby",
>     >     >                     "heartbeat-delay": 10000,
>     >     >                     "max-response-delay": 20000,
>     >     >                     "max-ack-delay": 10000,
>     >     >                     "max-unacked-clients": 0,
>     >     >                     "peers": [
>     >     >                         {
>     >     >                             "name": "KEA_DHCPv6_1",
>     >     >                             "role": "primary",
>     >     >                             "auto-failover": true
>     >     >                         },
>     >     >                         {
>     >     >                             "name": "KEA_DHCPv6_2",
>     >     >                             "role": "standby",
>     >     >                             "auto-failover": true
>     >     >                         }
>     >     >                     ]
>     >     >                 } ]
>     >     >             }
>     >     >         }
>     >     >     ]
>     >     > },
>     >     >
>     >     > "Logging":
>     >     > {
>     >     >   "loggers": [
>     >     >     {
>     >     >         "name": "kea-dhcp6",
>     >     >         "output_options": [
>     >     >             {
>     >     >                 "output": "/usr/local/var/log/kea-dhcp6.log"
>     >     >             }
>     >     >         ],
>     >     >         "severity": "INFO",
>     >     >
>     >     >         "debuglevel": 0
>     >     >     }
>     >     >   ]
>     >     > }
>     >     > }
>     >     > Thanks
>     >     >
>     >     > Nitzan
>     >     >
>     >     > _______________________________________________
>     >     > Kea-users mailing list
>     >     > [hidden email] <mailto:[hidden email]>
>     <mailto:[hidden email] <mailto:[hidden email]>>
>     >     > https://lists.isc.org/mailman/listinfo/kea-users
>     >     >
>     >
>


_______________________________________________
Kea-users mailing list
[hidden email]
https://lists.isc.org/mailman/listinfo/kea-users