Daily Archives: March 17, 2010

Shannon entropy is said additive in the sense that the entropy of the joint distribution

H(X*Y)

is the sum of the entropies:

H(X*Y) = H(X)+H(Y).

This property is not true for the quadratic entropy (sum of squares). The Java program below checks experimentally those entropy functional properties. Frank.

Here is the Java code

/*
 (C) March 2010, Frank Nielsen
 Demonstrates that Shannon entropy is additive
 and that the quadratic entropy is not
*/
public class ShannonAdditiveEntropy
{
 //quadratic entropy
public static double Q(double [] p)
{
double res=0.0d;
for(int i=0;i < p.length;i++)
    res+= p[i]*p[i];
    
return res;     
}   
 //Shannon entropy
public static double H(double [] p)
{
double res=0.0d;
for(int i=0;i < p.length;i++)
    res+= -p[i]*Math.log(p[i]);
    
return res;     
}   
    
public static void main (String [] args)
{
int n=10,i,j;
double sump=0.0, sumq=0.0;
double [] p=new double[n];
double [] q=new double[n];
double [] pq=new double[n*n];

for(i=0;i < n;i++)
{
p[i]=Math.random(); q[i]=Math.random(); 
sump+=p[i]; sumq+=q[i];
}
// normalize to densities
for(i=0;i < n;i++)
{
    p[i]/=sump;  q[i]/=sumq;
}

for(i=0;i < n;i++)
    for(j=0;j < n;j++)
     pq[i*n+j]=p[i]*q[j];
     
System.out.println("Show that Shannon entropy is an additive entropy:");
System.out.println("H(p)="+H(p)+" H(q)="+H(q)+" H(pq)="+H(pq) );
double delta=H(pq)-H(p)-H(q);
System.out.println("Additive entropy:"+delta);
System.out.println("Show that the quadratic entropy is NOT an additive entropy:");
System.out.println("Q(p)="+Q(p)+" Q(q)="+Q(q)+" H(pq)="+Q(pq) );
double deltaq=Q(pq)-Q(p)-Q(q);
System.out.println("Not an ddditive entropy:"+deltaq);
     
}
    
}

Frank.